liblttng-ust-libc-wrapper/lttng-ust-malloc.c

   1 /*
   2  * Copyright (C) 2009  Pierre-Marc Fournier
   3  * Copyright (C) 2011  Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   4  *
   5  * This library is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation; either
   8  * version 2.1 of the License, or (at your option) any later version.
   9  *
  10  * This library is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with this library; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  18  */
  19
  20 #define _GNU_SOURCE
  21 /*
  22  * Do _not_ define _LGPL_SOURCE because we don't want to create a
  23  * circular dependency loop between this malloc wrapper, liburcu and
  24  * libc.
  25  */
  26 #include <lttng/ust-dlfcn.h>
  27 #include <sys/types.h>
  28 #include <stdio.h>
  29 #include <assert.h>
  30 #include <urcu/system.h>
  31 #include <urcu/uatomic.h>
  32 #include <urcu/compiler.h>
  33 #include <urcu/tls-compat.h>
  34 #include <urcu/arch.h>
  35 #include <lttng/align.h>
  36 #include <helper.h>
  37
  38 #define TRACEPOINT_DEFINE
  39 #define TRACEPOINT_CREATE_PROBES
  40 #define TP_IP_PARAM ip
  41 #include "ust_libc.h"
  42
  43 #define STATIC_CALLOC_LEN 4096
  44 static char static_calloc_buf[STATIC_CALLOC_LEN];
  45 static unsigned long static_calloc_buf_offset;
  46
  47 struct alloc_functions {
  48         void *(*calloc)(size_t nmemb, size_t size);
  49         void *(*malloc)(size_t size);
  50         void (*free)(void *ptr);
  51         void *(*realloc)(void *ptr, size_t size);
  52         void *(*memalign)(size_t alignment, size_t size);
  53         int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
  54 };
  55
  56 static
  57 struct alloc_functions cur_alloc;
  58
  59 /*
  60  * Make sure our own use of the LTS compat layer will not cause infinite
  61  * recursion by calling calloc.
  62  */
  63
  64 static
  65 void *static_calloc(size_t nmemb, size_t size);
  66
  67 /*
  68  * pthread mutex replacement for URCU tls compat layer.
  69  */
  70 static int ust_malloc_lock;
  71
  72 static __attribute__((unused))
  73 void ust_malloc_spin_lock(pthread_mutex_t *lock)
  74 {
  75         /*
  76          * The memory barrier within cmpxchg takes care of ordering
  77          * memory accesses with respect to the start of the critical
  78          * section.
  79          */
  80         while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
  81                 caa_cpu_relax();
  82 }
  83
  84 static __attribute__((unused))
  85 void ust_malloc_spin_unlock(pthread_mutex_t *lock)
  86 {
  87         /*
  88          * Ensure memory accesses within the critical section do not
  89          * leak outside.
  90          */
  91         cmm_smp_mb();
  92         uatomic_set(&ust_malloc_lock, 0);
  93 }
  94
  95 #define calloc static_calloc
  96 #define pthread_mutex_lock ust_malloc_spin_lock
  97 #define pthread_mutex_unlock ust_malloc_spin_unlock
  98 static DEFINE_URCU_TLS(int, malloc_nesting);
  99 #undef ust_malloc_spin_unlock
 100 #undef ust_malloc_spin_lock
 101 #undef calloc
 102
 103 /*
 104  * Static allocator to use when initially executing dlsym(). It keeps a
 105  * size_t value of each object size prior to the object.
 106  */
 107 static
 108 void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
 109 {
 110         size_t prev_offset, new_offset, res_offset, aligned_offset;
 111
 112         if (nmemb * size == 0) {
 113                 return NULL;
 114         }
 115
 116         /*
 117          * Protect static_calloc_buf_offset from concurrent updates
 118          * using a cmpxchg loop rather than a mutex to remove a
 119          * dependency on pthread. This will minimize the risk of bad
 120          * interaction between mutex and malloc instrumentation.
 121          */
 122         res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
 123         do {
 124                 prev_offset = res_offset;
 125                 aligned_offset = ALIGN(prev_offset + sizeof(size_t), alignment);
 126                 new_offset = aligned_offset + nmemb * size;
 127                 if (new_offset > sizeof(static_calloc_buf)) {
 128                         abort();
 129                 }
 130         } while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
 131                         prev_offset, new_offset)) != prev_offset);
 132         *(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
 133         return &static_calloc_buf[aligned_offset];
 134 }
 135
 136 static
 137 void *static_calloc(size_t nmemb, size_t size)
 138 {
 139         void *retval;
 140
 141         retval = static_calloc_aligned(nmemb, size, 1);
 142         return retval;
 143 }
 144
 145 static
 146 void *static_malloc(size_t size)
 147 {
 148         void *retval;
 149
 150         retval = static_calloc_aligned(1, size, 1);
 151         return retval;
 152 }
 153
 154 static
 155 void static_free(void *ptr)
 156 {
 157         /* no-op. */
 158 }
 159
 160 static
 161 void *static_realloc(void *ptr, size_t size)
 162 {
 163         size_t *old_size = NULL;
 164         void *retval;
 165
 166         if (size == 0) {
 167                 retval = NULL;
 168                 goto end;
 169         }
 170
 171         if (ptr) {
 172                 old_size = (size_t *) ptr - 1;
 173                 if (size <= *old_size) {
 174                         /* We can re-use the old entry. */
 175                         *old_size = size;
 176                         retval = ptr;
 177                         goto end;
 178                 }
 179         }
 180         /* We need to expand. Don't free previous memory location. */
 181         retval = static_calloc_aligned(1, size, 1);
 182         assert(retval);
 183         if (ptr)
 184                 memcpy(retval, ptr, *old_size);
 185 end:
 186         return retval;
 187 }
 188
 189 static
 190 void *static_memalign(size_t alignment, size_t size)
 191 {
 192         void *retval;
 193
 194         retval = static_calloc_aligned(1, size, alignment);
 195         return retval;
 196 }
 197
 198 static
 199 int static_posix_memalign(void **memptr, size_t alignment, size_t size)
 200 {
 201         void *ptr;
 202
 203         /* Check for power of 2, larger than void *. */
 204         if (alignment & (alignment - 1)
 205                         || alignment < sizeof(void *)
 206                         || alignment == 0) {
 207                 goto end;
 208         }
 209         ptr = static_calloc_aligned(1, size, alignment);
 210         *memptr = ptr;
 211 end:
 212         return 0;
 213 }
 214
 215 static
 216 void setup_static_allocator(void)
 217 {
 218         assert(cur_alloc.calloc == NULL);
 219         cur_alloc.calloc = static_calloc;
 220         assert(cur_alloc.malloc == NULL);
 221         cur_alloc.malloc = static_malloc;
 222         assert(cur_alloc.free == NULL);
 223         cur_alloc.free = static_free;
 224         assert(cur_alloc.realloc == NULL);
 225         cur_alloc.realloc = static_realloc;
 226         assert(cur_alloc.memalign == NULL);
 227         cur_alloc.memalign = static_memalign;
 228         assert(cur_alloc.posix_memalign == NULL);
 229         cur_alloc.posix_memalign = static_posix_memalign;
 230 }
 231
 232 static
 233 void lookup_all_symbols(void)
 234 {
 235         struct alloc_functions af;
 236
 237         /*
 238          * Temporarily redirect allocation functions to
 239          * static_calloc_aligned, and free function to static_free
 240          * (no-op), until the dlsym lookup has completed.
 241          */
 242         setup_static_allocator();
 243
 244         /* Perform the actual lookups */
 245         af.calloc = dlsym(RTLD_NEXT, "calloc");
 246         af.malloc = dlsym(RTLD_NEXT, "malloc");
 247         af.free = dlsym(RTLD_NEXT, "free");
 248         af.realloc = dlsym(RTLD_NEXT, "realloc");
 249         af.memalign = dlsym(RTLD_NEXT, "memalign");
 250         af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
 251
 252         /* Populate the new allocator functions */
 253         memcpy(&cur_alloc, &af, sizeof(cur_alloc));
 254 }
 255
 256 void *malloc(size_t size)
 257 {
 258         void *retval;
 259
 260         URCU_TLS(malloc_nesting)++;
 261         if (cur_alloc.malloc == NULL) {
 262                 lookup_all_symbols();
 263                 if (cur_alloc.malloc == NULL) {
 264                         fprintf(stderr, "mallocwrap: unable to find malloc\n");
 265                         abort();
 266                 }
 267         }
 268         retval = cur_alloc.malloc(size);
 269         if (URCU_TLS(malloc_nesting) == 1) {
 270                 tracepoint(lttng_ust_libc, malloc,
 271                         size, retval, LTTNG_UST_CALLER_IP());
 272         }
 273         URCU_TLS(malloc_nesting)--;
 274         return retval;
 275 }
 276
 277 void free(void *ptr)
 278 {
 279         URCU_TLS(malloc_nesting)++;
 280         /*
 281          * Check whether the memory was allocated with
 282          * static_calloc_align, in which case there is nothing to free.
 283          */
 284         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 285                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 286                 goto end;
 287         }
 288
 289         if (URCU_TLS(malloc_nesting) == 1) {
 290                 tracepoint(lttng_ust_libc, free,
 291                         ptr, LTTNG_UST_CALLER_IP());
 292         }
 293
 294         if (cur_alloc.free == NULL) {
 295                 lookup_all_symbols();
 296                 if (cur_alloc.free == NULL) {
 297                         fprintf(stderr, "mallocwrap: unable to find free\n");
 298                         abort();
 299                 }
 300         }
 301         cur_alloc.free(ptr);
 302 end:
 303         URCU_TLS(malloc_nesting)--;
 304 }
 305
 306 void *calloc(size_t nmemb, size_t size)
 307 {
 308         void *retval;
 309
 310         URCU_TLS(malloc_nesting)++;
 311         if (cur_alloc.calloc == NULL) {
 312                 lookup_all_symbols();
 313                 if (cur_alloc.calloc == NULL) {
 314                         fprintf(stderr, "callocwrap: unable to find calloc\n");
 315                         abort();
 316                 }
 317         }
 318         retval = cur_alloc.calloc(nmemb, size);
 319         if (URCU_TLS(malloc_nesting) == 1) {
 320                 tracepoint(lttng_ust_libc, calloc,
 321                         nmemb, size, retval, LTTNG_UST_CALLER_IP());
 322         }
 323         URCU_TLS(malloc_nesting)--;
 324         return retval;
 325 }
 326
 327 void *realloc(void *ptr, size_t size)
 328 {
 329         void *retval;
 330
 331         URCU_TLS(malloc_nesting)++;
 332         /*
 333          * Check whether the memory was allocated with
 334          * static_calloc_align, in which case there is nothing
 335          * to free, and we need to copy the old data.
 336          */
 337         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 338                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 339                 size_t *old_size;
 340
 341                 old_size = (size_t *) ptr - 1;
 342                 if (cur_alloc.calloc == NULL) {
 343                         lookup_all_symbols();
 344                         if (cur_alloc.calloc == NULL) {
 345                                 fprintf(stderr, "reallocwrap: unable to find calloc\n");
 346                                 abort();
 347                         }
 348                 }
 349                 retval = cur_alloc.calloc(1, size);
 350                 if (retval) {
 351                         memcpy(retval, ptr, *old_size);
 352                 }
 353                 /*
 354                  * Mimick that a NULL pointer has been received, so
 355                  * memory allocation analysis based on the trace don't
 356                  * get confused by the address from the static
 357                  * allocator.
 358                  */
 359                 ptr = NULL;
 360                 goto end;
 361         }
 362
 363         if (cur_alloc.realloc == NULL) {
 364                 lookup_all_symbols();
 365                 if (cur_alloc.realloc == NULL) {
 366                         fprintf(stderr, "reallocwrap: unable to find realloc\n");
 367                         abort();
 368                 }
 369         }
 370         retval = cur_alloc.realloc(ptr, size);
 371 end:
 372         if (URCU_TLS(malloc_nesting) == 1) {
 373                 tracepoint(lttng_ust_libc, realloc,
 374                         ptr, size, retval, LTTNG_UST_CALLER_IP());
 375         }
 376         URCU_TLS(malloc_nesting)--;
 377         return retval;
 378 }
 379
 380 void *memalign(size_t alignment, size_t size)
 381 {
 382         void *retval;
 383
 384         URCU_TLS(malloc_nesting)++;
 385         if (cur_alloc.memalign == NULL) {
 386                 lookup_all_symbols();
 387                 if (cur_alloc.memalign == NULL) {
 388                         fprintf(stderr, "memalignwrap: unable to find memalign\n");
 389                         abort();
 390                 }
 391         }
 392         retval = cur_alloc.memalign(alignment, size);
 393         if (URCU_TLS(malloc_nesting) == 1) {
 394                 tracepoint(lttng_ust_libc, memalign,
 395                         alignment, size, retval,
 396                         LTTNG_UST_CALLER_IP());
 397         }
 398         URCU_TLS(malloc_nesting)--;
 399         return retval;
 400 }
 401
 402 int posix_memalign(void **memptr, size_t alignment, size_t size)
 403 {
 404         int retval;
 405
 406         URCU_TLS(malloc_nesting)++;
 407         if (cur_alloc.posix_memalign == NULL) {
 408                 lookup_all_symbols();
 409                 if (cur_alloc.posix_memalign == NULL) {
 410                         fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
 411                         abort();
 412                 }
 413         }
 414         retval = cur_alloc.posix_memalign(memptr, alignment, size);
 415         if (URCU_TLS(malloc_nesting) == 1) {
 416                 tracepoint(lttng_ust_libc, posix_memalign,
 417                         *memptr, alignment, size,
 418                         retval, LTTNG_UST_CALLER_IP());
 419         }
 420         URCU_TLS(malloc_nesting)--;
 421         return retval;
 422 }
 423
 424 static
 425 void lttng_ust_fixup_malloc_nesting_tls(void)
 426 {
 427         asm volatile ("" : : "m" (URCU_TLS(malloc_nesting)));
 428 }
 429
 430 __attribute__((constructor))
 431 void lttng_ust_malloc_wrapper_init(void)
 432 {
 433         /* Initialization already done */
 434         if (cur_alloc.calloc) {
 435                 return;
 436         }
 437         lttng_ust_fixup_malloc_nesting_tls();
 438         /*
 439          * Ensure the allocator is in place before the process becomes
 440          * multithreaded.
 441          */
 442         lookup_all_symbols();
 443 }