src/lib/lttng-ust-libc-wrapper/lttng-ust-malloc.c

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-or-later
   3  *
   4  * Copyright (C) 2009 Pierre-Marc Fournier
   5  * Copyright (C) 2011 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 /*
   9  * Do _not_ define _LGPL_SOURCE because we don't want to create a
  10  * circular dependency loop between this malloc wrapper, liburcu and
  11  * libc.
  12  */
  13
  14 /* Has to be included first to override dlfcn.h */
  15 #include <common/compat/dlfcn.h>
  16
  17 #include <sys/types.h>
  18 #include <stdio.h>
  19 #include <assert.h>
  20 #include <malloc.h>
  21
  22 #include <urcu/system.h>
  23 #include <urcu/uatomic.h>
  24 #include <urcu/compiler.h>
  25 #include <urcu/tls-compat.h>
  26 #include <urcu/arch.h>
  27
  28 #include <lttng/ust-libc-wrapper.h>
  29
  30 #include "common/macros.h"
  31 #include "common/align.h"
  32
  33 #define LTTNG_UST_TRACEPOINT_HIDDEN_DEFINITION
  34 #define LTTNG_UST_TRACEPOINT_PROVIDER_HIDDEN_DEFINITION
  35
  36 #define LTTNG_UST_TRACEPOINT_DEFINE
  37 #define LTTNG_UST_TRACEPOINT_CREATE_PROBES
  38 #define LTTNG_UST_TP_IP_PARAM ip
  39 #include "ust_libc.h"
  40
  41 #define STATIC_CALLOC_LEN 4096
  42 static char static_calloc_buf[STATIC_CALLOC_LEN];
  43 static unsigned long static_calloc_buf_offset;
  44
  45 struct alloc_functions {
  46         void *(*calloc)(size_t nmemb, size_t size);
  47         void *(*malloc)(size_t size);
  48         void (*free)(void *ptr);
  49         void *(*realloc)(void *ptr, size_t size);
  50         void *(*memalign)(size_t alignment, size_t size);
  51         int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
  52 };
  53
  54 static
  55 struct alloc_functions cur_alloc;
  56
  57 /*
  58  * Make sure our own use of the LTS compat layer will not cause infinite
  59  * recursion by calling calloc.
  60  */
  61
  62 static
  63 void *static_calloc(size_t nmemb, size_t size);
  64
  65 /*
  66  * pthread mutex replacement for URCU tls compat layer.
  67  */
  68 static int ust_malloc_lock;
  69
  70 static
  71 void ust_malloc_spin_lock(pthread_mutex_t *lock)
  72         __attribute__((unused));
  73 static
  74 void ust_malloc_spin_lock(pthread_mutex_t *lock __attribute__((unused)))
  75 {
  76         /*
  77          * The memory barrier within cmpxchg takes care of ordering
  78          * memory accesses with respect to the start of the critical
  79          * section.
  80          */
  81         while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
  82                 caa_cpu_relax();
  83 }
  84
  85 static
  86 void ust_malloc_spin_unlock(pthread_mutex_t *lock)
  87         __attribute__((unused));
  88 static
  89 void ust_malloc_spin_unlock(pthread_mutex_t *lock __attribute__((unused)))
  90 {
  91         /*
  92          * Ensure memory accesses within the critical section do not
  93          * leak outside.
  94          */
  95         cmm_smp_mb();
  96         uatomic_set(&ust_malloc_lock, 0);
  97 }
  98
  99 #define calloc static_calloc
 100 #define pthread_mutex_lock ust_malloc_spin_lock
 101 #define pthread_mutex_unlock ust_malloc_spin_unlock
 102 static DEFINE_URCU_TLS(int, malloc_nesting);
 103 #undef pthread_mutex_unlock
 104 #undef pthread_mutex_lock
 105 #undef calloc
 106
 107 /*
 108  * Static allocator to use when initially executing dlsym(). It keeps a
 109  * size_t value of each object size prior to the object.
 110  */
 111 static
 112 void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
 113 {
 114         size_t prev_offset, new_offset, res_offset, aligned_offset;
 115
 116         if (nmemb * size == 0) {
 117                 return NULL;
 118         }
 119
 120         /*
 121          * Protect static_calloc_buf_offset from concurrent updates
 122          * using a cmpxchg loop rather than a mutex to remove a
 123          * dependency on pthread. This will minimize the risk of bad
 124          * interaction between mutex and malloc instrumentation.
 125          */
 126         res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
 127         do {
 128                 prev_offset = res_offset;
 129                 aligned_offset = LTTNG_UST_ALIGN(prev_offset + sizeof(size_t), alignment);
 130                 new_offset = aligned_offset + nmemb * size;
 131                 if (new_offset > sizeof(static_calloc_buf)) {
 132                         abort();
 133                 }
 134         } while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
 135                         prev_offset, new_offset)) != prev_offset);
 136         *(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
 137         return &static_calloc_buf[aligned_offset];
 138 }
 139
 140 static
 141 void *static_calloc(size_t nmemb, size_t size)
 142 {
 143         void *retval;
 144
 145         retval = static_calloc_aligned(nmemb, size, 1);
 146         return retval;
 147 }
 148
 149 static
 150 void *static_malloc(size_t size)
 151 {
 152         void *retval;
 153
 154         retval = static_calloc_aligned(1, size, 1);
 155         return retval;
 156 }
 157
 158 static
 159 void static_free(void *ptr __attribute__((unused)))
 160 {
 161         /* no-op. */
 162 }
 163
 164 static
 165 void *static_realloc(void *ptr, size_t size)
 166 {
 167         size_t *old_size = NULL;
 168         void *retval;
 169
 170         if (size == 0) {
 171                 retval = NULL;
 172                 goto end;
 173         }
 174
 175         if (ptr) {
 176                 old_size = (size_t *) ptr - 1;
 177                 if (size <= *old_size) {
 178                         /* We can re-use the old entry. */
 179                         *old_size = size;
 180                         retval = ptr;
 181                         goto end;
 182                 }
 183         }
 184         /* We need to expand. Don't free previous memory location. */
 185         retval = static_calloc_aligned(1, size, 1);
 186         assert(retval);
 187         if (ptr)
 188                 memcpy(retval, ptr, *old_size);
 189 end:
 190         return retval;
 191 }
 192
 193 static
 194 void *static_memalign(size_t alignment, size_t size)
 195 {
 196         void *retval;
 197
 198         retval = static_calloc_aligned(1, size, alignment);
 199         return retval;
 200 }
 201
 202 static
 203 int static_posix_memalign(void **memptr, size_t alignment, size_t size)
 204 {
 205         void *ptr;
 206
 207         /* Check for power of 2, larger than void *. */
 208         if (alignment & (alignment - 1)
 209                         || alignment < sizeof(void *)
 210                         || alignment == 0) {
 211                 goto end;
 212         }
 213         ptr = static_calloc_aligned(1, size, alignment);
 214         *memptr = ptr;
 215 end:
 216         return 0;
 217 }
 218
 219 static
 220 void setup_static_allocator(void)
 221 {
 222         assert(cur_alloc.calloc == NULL);
 223         cur_alloc.calloc = static_calloc;
 224         assert(cur_alloc.malloc == NULL);
 225         cur_alloc.malloc = static_malloc;
 226         assert(cur_alloc.free == NULL);
 227         cur_alloc.free = static_free;
 228         assert(cur_alloc.realloc == NULL);
 229         cur_alloc.realloc = static_realloc;
 230         assert(cur_alloc.memalign == NULL);
 231         cur_alloc.memalign = static_memalign;
 232         assert(cur_alloc.posix_memalign == NULL);
 233         cur_alloc.posix_memalign = static_posix_memalign;
 234 }
 235
 236 static
 237 void lookup_all_symbols(void)
 238 {
 239         struct alloc_functions af;
 240
 241         /*
 242          * Temporarily redirect allocation functions to
 243          * static_calloc_aligned, and free function to static_free
 244          * (no-op), until the dlsym lookup has completed.
 245          */
 246         setup_static_allocator();
 247
 248         /* Perform the actual lookups */
 249         af.calloc = dlsym(RTLD_NEXT, "calloc");
 250         af.malloc = dlsym(RTLD_NEXT, "malloc");
 251         af.free = dlsym(RTLD_NEXT, "free");
 252         af.realloc = dlsym(RTLD_NEXT, "realloc");
 253         af.memalign = dlsym(RTLD_NEXT, "memalign");
 254         af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
 255
 256         /* Populate the new allocator functions */
 257         memcpy(&cur_alloc, &af, sizeof(cur_alloc));
 258 }
 259
 260 void *malloc(size_t size)
 261 {
 262         void *retval;
 263
 264         URCU_TLS(malloc_nesting)++;
 265         if (cur_alloc.malloc == NULL) {
 266                 lookup_all_symbols();
 267                 if (cur_alloc.malloc == NULL) {
 268                         fprintf(stderr, "mallocwrap: unable to find malloc\n");
 269                         abort();
 270                 }
 271         }
 272         retval = cur_alloc.malloc(size);
 273         if (URCU_TLS(malloc_nesting) == 1) {
 274                 lttng_ust_tracepoint(lttng_ust_libc, malloc,
 275                         size, retval, LTTNG_UST_CALLER_IP());
 276         }
 277         URCU_TLS(malloc_nesting)--;
 278         return retval;
 279 }
 280
 281 void free(void *ptr)
 282 {
 283         URCU_TLS(malloc_nesting)++;
 284         /*
 285          * Check whether the memory was allocated with
 286          * static_calloc_align, in which case there is nothing to free.
 287          */
 288         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 289                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 290                 goto end;
 291         }
 292
 293         if (URCU_TLS(malloc_nesting) == 1) {
 294                 lttng_ust_tracepoint(lttng_ust_libc, free,
 295                         ptr, LTTNG_UST_CALLER_IP());
 296         }
 297
 298         if (cur_alloc.free == NULL) {
 299                 lookup_all_symbols();
 300                 if (cur_alloc.free == NULL) {
 301                         fprintf(stderr, "mallocwrap: unable to find free\n");
 302                         abort();
 303                 }
 304         }
 305         cur_alloc.free(ptr);
 306 end:
 307         URCU_TLS(malloc_nesting)--;
 308 }
 309
 310 void *calloc(size_t nmemb, size_t size)
 311 {
 312         void *retval;
 313
 314         URCU_TLS(malloc_nesting)++;
 315         if (cur_alloc.calloc == NULL) {
 316                 lookup_all_symbols();
 317                 if (cur_alloc.calloc == NULL) {
 318                         fprintf(stderr, "callocwrap: unable to find calloc\n");
 319                         abort();
 320                 }
 321         }
 322         retval = cur_alloc.calloc(nmemb, size);
 323         if (URCU_TLS(malloc_nesting) == 1) {
 324                 lttng_ust_tracepoint(lttng_ust_libc, calloc,
 325                         nmemb, size, retval, LTTNG_UST_CALLER_IP());
 326         }
 327         URCU_TLS(malloc_nesting)--;
 328         return retval;
 329 }
 330
 331 void *realloc(void *ptr, size_t size)
 332 {
 333         void *retval;
 334
 335         URCU_TLS(malloc_nesting)++;
 336         /*
 337          * Check whether the memory was allocated with
 338          * static_calloc_align, in which case there is nothing
 339          * to free, and we need to copy the old data.
 340          */
 341         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 342                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 343                 size_t *old_size;
 344
 345                 old_size = (size_t *) ptr - 1;
 346                 if (cur_alloc.calloc == NULL) {
 347                         lookup_all_symbols();
 348                         if (cur_alloc.calloc == NULL) {
 349                                 fprintf(stderr, "reallocwrap: unable to find calloc\n");
 350                                 abort();
 351                         }
 352                 }
 353                 retval = cur_alloc.calloc(1, size);
 354                 if (retval) {
 355                         memcpy(retval, ptr, *old_size);
 356                 }
 357                 /*
 358                  * Mimick that a NULL pointer has been received, so
 359                  * memory allocation analysis based on the trace don't
 360                  * get confused by the address from the static
 361                  * allocator.
 362                  */
 363                 ptr = NULL;
 364                 goto end;
 365         }
 366
 367         if (cur_alloc.realloc == NULL) {
 368                 lookup_all_symbols();
 369                 if (cur_alloc.realloc == NULL) {
 370                         fprintf(stderr, "reallocwrap: unable to find realloc\n");
 371                         abort();
 372                 }
 373         }
 374         retval = cur_alloc.realloc(ptr, size);
 375 end:
 376         if (URCU_TLS(malloc_nesting) == 1) {
 377                 lttng_ust_tracepoint(lttng_ust_libc, realloc,
 378                         ptr, size, retval, LTTNG_UST_CALLER_IP());
 379         }
 380         URCU_TLS(malloc_nesting)--;
 381         return retval;
 382 }
 383
 384 void *memalign(size_t alignment, size_t size)
 385 {
 386         void *retval;
 387
 388         URCU_TLS(malloc_nesting)++;
 389         if (cur_alloc.memalign == NULL) {
 390                 lookup_all_symbols();
 391                 if (cur_alloc.memalign == NULL) {
 392                         fprintf(stderr, "memalignwrap: unable to find memalign\n");
 393                         abort();
 394                 }
 395         }
 396         retval = cur_alloc.memalign(alignment, size);
 397         if (URCU_TLS(malloc_nesting) == 1) {
 398                 lttng_ust_tracepoint(lttng_ust_libc, memalign,
 399                         alignment, size, retval,
 400                         LTTNG_UST_CALLER_IP());
 401         }
 402         URCU_TLS(malloc_nesting)--;
 403         return retval;
 404 }
 405
 406 int posix_memalign(void **memptr, size_t alignment, size_t size)
 407 {
 408         int retval;
 409
 410         URCU_TLS(malloc_nesting)++;
 411         if (cur_alloc.posix_memalign == NULL) {
 412                 lookup_all_symbols();
 413                 if (cur_alloc.posix_memalign == NULL) {
 414                         fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
 415                         abort();
 416                 }
 417         }
 418         retval = cur_alloc.posix_memalign(memptr, alignment, size);
 419         if (URCU_TLS(malloc_nesting) == 1) {
 420                 lttng_ust_tracepoint(lttng_ust_libc, posix_memalign,
 421                         *memptr, alignment, size,
 422                         retval, LTTNG_UST_CALLER_IP());
 423         }
 424         URCU_TLS(malloc_nesting)--;
 425         return retval;
 426 }
 427
 428 static
 429 void lttng_ust_fixup_malloc_nesting_tls(void)
 430 {
 431         asm volatile ("" : : "m" (URCU_TLS(malloc_nesting)));
 432 }
 433
 434 void lttng_ust_libc_wrapper_malloc_ctor(void)
 435 {
 436         /* Initialization already done */
 437         if (cur_alloc.calloc) {
 438                 return;
 439         }
 440         lttng_ust_fixup_malloc_nesting_tls();
 441         /*
 442          * Ensure the allocator is in place before the process becomes
 443          * multithreaded.
 444          */
 445         lookup_all_symbols();
 446 }