liblttng-ust-libc-wrapper/lttng-ust-malloc.c

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-or-later
   3  *
   4  * Copyright (C) 2009 Pierre-Marc Fournier
   5  * Copyright (C) 2011 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 /*
   9  * Do _not_ define _LGPL_SOURCE because we don't want to create a
  10  * circular dependency loop between this malloc wrapper, liburcu and
  11  * libc.
  12  */
  13 #include <lttng/ust-dlfcn.h>
  14 #include <sys/types.h>
  15 #include <stdio.h>
  16 #include <assert.h>
  17 #include <urcu/system.h>
  18 #include <urcu/uatomic.h>
  19 #include <urcu/compiler.h>
  20 #include <urcu/tls-compat.h>
  21 #include <urcu/arch.h>
  22 #include <lttng/align.h>
  23 #include <helper.h>
  24
  25 #define TRACEPOINT_DEFINE
  26 #define TRACEPOINT_CREATE_PROBES
  27 #define TP_IP_PARAM ip
  28 #include "ust_libc.h"
  29
  30 #define STATIC_CALLOC_LEN 4096
  31 static char static_calloc_buf[STATIC_CALLOC_LEN];
  32 static unsigned long static_calloc_buf_offset;
  33
  34 struct alloc_functions {
  35         void *(*calloc)(size_t nmemb, size_t size);
  36         void *(*malloc)(size_t size);
  37         void (*free)(void *ptr);
  38         void *(*realloc)(void *ptr, size_t size);
  39         void *(*memalign)(size_t alignment, size_t size);
  40         int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
  41 };
  42
  43 static
  44 struct alloc_functions cur_alloc;
  45
  46 /*
  47  * Make sure our own use of the LTS compat layer will not cause infinite
  48  * recursion by calling calloc.
  49  */
  50
  51 static
  52 void *static_calloc(size_t nmemb, size_t size);
  53
  54 /*
  55  * pthread mutex replacement for URCU tls compat layer.
  56  */
  57 static int ust_malloc_lock;
  58
  59 static __attribute__((unused))
  60 void ust_malloc_spin_lock(pthread_mutex_t *lock)
  61 {
  62         /*
  63          * The memory barrier within cmpxchg takes care of ordering
  64          * memory accesses with respect to the start of the critical
  65          * section.
  66          */
  67         while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
  68                 caa_cpu_relax();
  69 }
  70
  71 static __attribute__((unused))
  72 void ust_malloc_spin_unlock(pthread_mutex_t *lock)
  73 {
  74         /*
  75          * Ensure memory accesses within the critical section do not
  76          * leak outside.
  77          */
  78         cmm_smp_mb();
  79         uatomic_set(&ust_malloc_lock, 0);
  80 }
  81
  82 #define calloc static_calloc
  83 #define pthread_mutex_lock ust_malloc_spin_lock
  84 #define pthread_mutex_unlock ust_malloc_spin_unlock
  85 static DEFINE_URCU_TLS(int, malloc_nesting);
  86 #undef pthread_mutex_unlock
  87 #undef pthread_mutex_lock
  88 #undef calloc
  89
  90 /*
  91  * Static allocator to use when initially executing dlsym(). It keeps a
  92  * size_t value of each object size prior to the object.
  93  */
  94 static
  95 void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
  96 {
  97         size_t prev_offset, new_offset, res_offset, aligned_offset;
  98
  99         if (nmemb * size == 0) {
 100                 return NULL;
 101         }
 102
 103         /*
 104          * Protect static_calloc_buf_offset from concurrent updates
 105          * using a cmpxchg loop rather than a mutex to remove a
 106          * dependency on pthread. This will minimize the risk of bad
 107          * interaction between mutex and malloc instrumentation.
 108          */
 109         res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
 110         do {
 111                 prev_offset = res_offset;
 112                 aligned_offset = LTTNG_UST_ALIGN(prev_offset + sizeof(size_t), alignment);
 113                 new_offset = aligned_offset + nmemb * size;
 114                 if (new_offset > sizeof(static_calloc_buf)) {
 115                         abort();
 116                 }
 117         } while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
 118                         prev_offset, new_offset)) != prev_offset);
 119         *(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
 120         return &static_calloc_buf[aligned_offset];
 121 }
 122
 123 static
 124 void *static_calloc(size_t nmemb, size_t size)
 125 {
 126         void *retval;
 127
 128         retval = static_calloc_aligned(nmemb, size, 1);
 129         return retval;
 130 }
 131
 132 static
 133 void *static_malloc(size_t size)
 134 {
 135         void *retval;
 136
 137         retval = static_calloc_aligned(1, size, 1);
 138         return retval;
 139 }
 140
 141 static
 142 void static_free(void *ptr)
 143 {
 144         /* no-op. */
 145 }
 146
 147 static
 148 void *static_realloc(void *ptr, size_t size)
 149 {
 150         size_t *old_size = NULL;
 151         void *retval;
 152
 153         if (size == 0) {
 154                 retval = NULL;
 155                 goto end;
 156         }
 157
 158         if (ptr) {
 159                 old_size = (size_t *) ptr - 1;
 160                 if (size <= *old_size) {
 161                         /* We can re-use the old entry. */
 162                         *old_size = size;
 163                         retval = ptr;
 164                         goto end;
 165                 }
 166         }
 167         /* We need to expand. Don't free previous memory location. */
 168         retval = static_calloc_aligned(1, size, 1);
 169         assert(retval);
 170         if (ptr)
 171                 memcpy(retval, ptr, *old_size);
 172 end:
 173         return retval;
 174 }
 175
 176 static
 177 void *static_memalign(size_t alignment, size_t size)
 178 {
 179         void *retval;
 180
 181         retval = static_calloc_aligned(1, size, alignment);
 182         return retval;
 183 }
 184
 185 static
 186 int static_posix_memalign(void **memptr, size_t alignment, size_t size)
 187 {
 188         void *ptr;
 189
 190         /* Check for power of 2, larger than void *. */
 191         if (alignment & (alignment - 1)
 192                         || alignment < sizeof(void *)
 193                         || alignment == 0) {
 194                 goto end;
 195         }
 196         ptr = static_calloc_aligned(1, size, alignment);
 197         *memptr = ptr;
 198 end:
 199         return 0;
 200 }
 201
 202 static
 203 void setup_static_allocator(void)
 204 {
 205         assert(cur_alloc.calloc == NULL);
 206         cur_alloc.calloc = static_calloc;
 207         assert(cur_alloc.malloc == NULL);
 208         cur_alloc.malloc = static_malloc;
 209         assert(cur_alloc.free == NULL);
 210         cur_alloc.free = static_free;
 211         assert(cur_alloc.realloc == NULL);
 212         cur_alloc.realloc = static_realloc;
 213         assert(cur_alloc.memalign == NULL);
 214         cur_alloc.memalign = static_memalign;
 215         assert(cur_alloc.posix_memalign == NULL);
 216         cur_alloc.posix_memalign = static_posix_memalign;
 217 }
 218
 219 static
 220 void lookup_all_symbols(void)
 221 {
 222         struct alloc_functions af;
 223
 224         /*
 225          * Temporarily redirect allocation functions to
 226          * static_calloc_aligned, and free function to static_free
 227          * (no-op), until the dlsym lookup has completed.
 228          */
 229         setup_static_allocator();
 230
 231         /* Perform the actual lookups */
 232         af.calloc = dlsym(RTLD_NEXT, "calloc");
 233         af.malloc = dlsym(RTLD_NEXT, "malloc");
 234         af.free = dlsym(RTLD_NEXT, "free");
 235         af.realloc = dlsym(RTLD_NEXT, "realloc");
 236         af.memalign = dlsym(RTLD_NEXT, "memalign");
 237         af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
 238
 239         /* Populate the new allocator functions */
 240         memcpy(&cur_alloc, &af, sizeof(cur_alloc));
 241 }
 242
 243 void *malloc(size_t size)
 244 {
 245         void *retval;
 246
 247         URCU_TLS(malloc_nesting)++;
 248         if (cur_alloc.malloc == NULL) {
 249                 lookup_all_symbols();
 250                 if (cur_alloc.malloc == NULL) {
 251                         fprintf(stderr, "mallocwrap: unable to find malloc\n");
 252                         abort();
 253                 }
 254         }
 255         retval = cur_alloc.malloc(size);
 256         if (URCU_TLS(malloc_nesting) == 1) {
 257                 tracepoint(lttng_ust_libc, malloc,
 258                         size, retval, LTTNG_UST_CALLER_IP());
 259         }
 260         URCU_TLS(malloc_nesting)--;
 261         return retval;
 262 }
 263
 264 void free(void *ptr)
 265 {
 266         URCU_TLS(malloc_nesting)++;
 267         /*
 268          * Check whether the memory was allocated with
 269          * static_calloc_align, in which case there is nothing to free.
 270          */
 271         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 272                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 273                 goto end;
 274         }
 275
 276         if (URCU_TLS(malloc_nesting) == 1) {
 277                 tracepoint(lttng_ust_libc, free,
 278                         ptr, LTTNG_UST_CALLER_IP());
 279         }
 280
 281         if (cur_alloc.free == NULL) {
 282                 lookup_all_symbols();
 283                 if (cur_alloc.free == NULL) {
 284                         fprintf(stderr, "mallocwrap: unable to find free\n");
 285                         abort();
 286                 }
 287         }
 288         cur_alloc.free(ptr);
 289 end:
 290         URCU_TLS(malloc_nesting)--;
 291 }
 292
 293 void *calloc(size_t nmemb, size_t size)
 294 {
 295         void *retval;
 296
 297         URCU_TLS(malloc_nesting)++;
 298         if (cur_alloc.calloc == NULL) {
 299                 lookup_all_symbols();
 300                 if (cur_alloc.calloc == NULL) {
 301                         fprintf(stderr, "callocwrap: unable to find calloc\n");
 302                         abort();
 303                 }
 304         }
 305         retval = cur_alloc.calloc(nmemb, size);
 306         if (URCU_TLS(malloc_nesting) == 1) {
 307                 tracepoint(lttng_ust_libc, calloc,
 308                         nmemb, size, retval, LTTNG_UST_CALLER_IP());
 309         }
 310         URCU_TLS(malloc_nesting)--;
 311         return retval;
 312 }
 313
 314 void *realloc(void *ptr, size_t size)
 315 {
 316         void *retval;
 317
 318         URCU_TLS(malloc_nesting)++;
 319         /*
 320          * Check whether the memory was allocated with
 321          * static_calloc_align, in which case there is nothing
 322          * to free, and we need to copy the old data.
 323          */
 324         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 325                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 326                 size_t *old_size;
 327
 328                 old_size = (size_t *) ptr - 1;
 329                 if (cur_alloc.calloc == NULL) {
 330                         lookup_all_symbols();
 331                         if (cur_alloc.calloc == NULL) {
 332                                 fprintf(stderr, "reallocwrap: unable to find calloc\n");
 333                                 abort();
 334                         }
 335                 }
 336                 retval = cur_alloc.calloc(1, size);
 337                 if (retval) {
 338                         memcpy(retval, ptr, *old_size);
 339                 }
 340                 /*
 341                  * Mimick that a NULL pointer has been received, so
 342                  * memory allocation analysis based on the trace don't
 343                  * get confused by the address from the static
 344                  * allocator.
 345                  */
 346                 ptr = NULL;
 347                 goto end;
 348         }
 349
 350         if (cur_alloc.realloc == NULL) {
 351                 lookup_all_symbols();
 352                 if (cur_alloc.realloc == NULL) {
 353                         fprintf(stderr, "reallocwrap: unable to find realloc\n");
 354                         abort();
 355                 }
 356         }
 357         retval = cur_alloc.realloc(ptr, size);
 358 end:
 359         if (URCU_TLS(malloc_nesting) == 1) {
 360                 tracepoint(lttng_ust_libc, realloc,
 361                         ptr, size, retval, LTTNG_UST_CALLER_IP());
 362         }
 363         URCU_TLS(malloc_nesting)--;
 364         return retval;
 365 }
 366
 367 void *memalign(size_t alignment, size_t size)
 368 {
 369         void *retval;
 370
 371         URCU_TLS(malloc_nesting)++;
 372         if (cur_alloc.memalign == NULL) {
 373                 lookup_all_symbols();
 374                 if (cur_alloc.memalign == NULL) {
 375                         fprintf(stderr, "memalignwrap: unable to find memalign\n");
 376                         abort();
 377                 }
 378         }
 379         retval = cur_alloc.memalign(alignment, size);
 380         if (URCU_TLS(malloc_nesting) == 1) {
 381                 tracepoint(lttng_ust_libc, memalign,
 382                         alignment, size, retval,
 383                         LTTNG_UST_CALLER_IP());
 384         }
 385         URCU_TLS(malloc_nesting)--;
 386         return retval;
 387 }
 388
 389 int posix_memalign(void **memptr, size_t alignment, size_t size)
 390 {
 391         int retval;
 392
 393         URCU_TLS(malloc_nesting)++;
 394         if (cur_alloc.posix_memalign == NULL) {
 395                 lookup_all_symbols();
 396                 if (cur_alloc.posix_memalign == NULL) {
 397                         fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
 398                         abort();
 399                 }
 400         }
 401         retval = cur_alloc.posix_memalign(memptr, alignment, size);
 402         if (URCU_TLS(malloc_nesting) == 1) {
 403                 tracepoint(lttng_ust_libc, posix_memalign,
 404                         *memptr, alignment, size,
 405                         retval, LTTNG_UST_CALLER_IP());
 406         }
 407         URCU_TLS(malloc_nesting)--;
 408         return retval;
 409 }
 410
 411 static
 412 void lttng_ust_fixup_malloc_nesting_tls(void)
 413 {
 414         asm volatile ("" : : "m" (URCU_TLS(malloc_nesting)));
 415 }
 416
 417 __attribute__((constructor))
 418 void lttng_ust_malloc_wrapper_init(void)
 419 {
 420         /* Initialization already done */
 421         if (cur_alloc.calloc) {
 422                 return;
 423         }
 424         lttng_ust_fixup_malloc_nesting_tls();
 425         /*
 426          * Ensure the allocator is in place before the process becomes
 427          * multithreaded.
 428          */
 429         lookup_all_symbols();
 430 }