liblttng-ust-libc-wrapper/lttng-ust-malloc.c

   1 /*
   2  * Copyright (C) 2009  Pierre-Marc Fournier
   3  * Copyright (C) 2011  Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   4  *
   5  * This library is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation; either
   8  * version 2.1 of the License, or (at your option) any later version.
   9  *
  10  * This library is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with this library; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  18  */
  19
  20 #define _GNU_SOURCE
  21 #include <lttng/ust-dlfcn.h>
  22 #include <sys/types.h>
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <urcu/system.h>
  26 #include <urcu/uatomic.h>
  27 #include <urcu/compiler.h>
  28 #include <urcu/tls-compat.h>
  29 #include <urcu/arch.h>
  30 #include <lttng/align.h>
  31
  32 #define TRACEPOINT_DEFINE
  33 #define TRACEPOINT_CREATE_PROBES
  34 #include "ust_libc.h"
  35
  36 #define STATIC_CALLOC_LEN 4096
  37 static char static_calloc_buf[STATIC_CALLOC_LEN];
  38 static unsigned long static_calloc_buf_offset;
  39
  40 struct alloc_functions {
  41         void *(*calloc)(size_t nmemb, size_t size);
  42         void *(*malloc)(size_t size);
  43         void (*free)(void *ptr);
  44         void *(*realloc)(void *ptr, size_t size);
  45         void *(*memalign)(size_t alignment, size_t size);
  46         int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
  47 };
  48
  49 static
  50 struct alloc_functions cur_alloc;
  51
  52 /*
  53  * Make sure our own use of the LTS compat layer will not cause infinite
  54  * recursion by calling calloc.
  55  */
  56
  57 static
  58 void *static_calloc(size_t nmemb, size_t size);
  59
  60 /*
  61  * pthread mutex replacement for URCU tls compat layer.
  62  */
  63 static int ust_malloc_lock;
  64
  65 static __attribute__((unused))
  66 void ust_malloc_spin_lock(pthread_mutex_t *lock)
  67 {
  68         /*
  69          * The memory barrier within cmpxchg takes care of ordering
  70          * memory accesses with respect to the start of the critical
  71          * section.
  72          */
  73         while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
  74                 caa_cpu_relax();
  75 }
  76
  77 static __attribute__((unused))
  78 void ust_malloc_spin_unlock(pthread_mutex_t *lock)
  79 {
  80         /*
  81          * Ensure memory accesses within the critical section do not
  82          * leak outside.
  83          */
  84         cmm_smp_mb();
  85         uatomic_set(&ust_malloc_lock, 0);
  86 }
  87
  88 #define calloc static_calloc
  89 #define pthread_mutex_lock ust_malloc_spin_lock
  90 #define pthread_mutex_unlock ust_malloc_spin_unlock
  91 static DEFINE_URCU_TLS(int, malloc_nesting);
  92 #undef ust_malloc_spin_unlock
  93 #undef ust_malloc_spin_lock
  94 #undef calloc
  95
  96 /*
  97  * Static allocator to use when initially executing dlsym(). It keeps a
  98  * size_t value of each object size prior to the object.
  99  */
 100 static
 101 void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
 102 {
 103         size_t prev_offset, new_offset, res_offset, aligned_offset;
 104
 105         if (nmemb * size == 0) {
 106                 return NULL;
 107         }
 108
 109         /*
 110          * Protect static_calloc_buf_offset from concurrent updates
 111          * using a cmpxchg loop rather than a mutex to remove a
 112          * dependency on pthread. This will minimize the risk of bad
 113          * interaction between mutex and malloc instrumentation.
 114          */
 115         res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
 116         do {
 117                 prev_offset = res_offset;
 118                 aligned_offset = ALIGN(prev_offset + sizeof(size_t), alignment);
 119                 new_offset = aligned_offset + nmemb * size;
 120                 if (new_offset > sizeof(static_calloc_buf)) {
 121                         abort();
 122                 }
 123         } while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
 124                         prev_offset, new_offset)) != prev_offset);
 125         *(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
 126         return &static_calloc_buf[aligned_offset];
 127 }
 128
 129 static
 130 void *static_calloc(size_t nmemb, size_t size)
 131 {
 132         void *retval;
 133
 134         retval = static_calloc_aligned(nmemb, size, 1);
 135         return retval;
 136 }
 137
 138 static
 139 void *static_malloc(size_t size)
 140 {
 141         void *retval;
 142
 143         retval = static_calloc_aligned(1, size, 1);
 144         return retval;
 145 }
 146
 147 static
 148 void static_free(void *ptr)
 149 {
 150         /* no-op. */
 151 }
 152
 153 static
 154 void *static_realloc(void *ptr, size_t size)
 155 {
 156         size_t *old_size = NULL;
 157         void *retval;
 158
 159         if (size == 0) {
 160                 retval = NULL;
 161                 goto end;
 162         }
 163
 164         if (ptr) {
 165                 old_size = (size_t *) ptr - 1;
 166                 if (size <= *old_size) {
 167                         /* We can re-use the old entry. */
 168                         *old_size = size;
 169                         retval = ptr;
 170                         goto end;
 171                 }
 172         }
 173         /* We need to expand. Don't free previous memory location. */
 174         retval = static_calloc_aligned(1, size, 1);
 175         assert(retval);
 176         if (ptr)
 177                 memcpy(retval, ptr, *old_size);
 178 end:
 179         return retval;
 180 }
 181
 182 static
 183 void *static_memalign(size_t alignment, size_t size)
 184 {
 185         void *retval;
 186
 187         retval = static_calloc_aligned(1, size, alignment);
 188         return retval;
 189 }
 190
 191 static
 192 int static_posix_memalign(void **memptr, size_t alignment, size_t size)
 193 {
 194         void *ptr;
 195
 196         /* Check for power of 2, larger than void *. */
 197         if (alignment & (alignment - 1)
 198                         || alignment < sizeof(void *)
 199                         || alignment == 0) {
 200                 goto end;
 201         }
 202         ptr = static_calloc_aligned(1, size, alignment);
 203         *memptr = ptr;
 204 end:
 205         return 0;
 206 }
 207
 208 static
 209 void setup_static_allocator(void)
 210 {
 211         assert(cur_alloc.calloc == NULL);
 212         cur_alloc.calloc = static_calloc;
 213         assert(cur_alloc.malloc == NULL);
 214         cur_alloc.malloc = static_malloc;
 215         assert(cur_alloc.free == NULL);
 216         cur_alloc.free = static_free;
 217         assert(cur_alloc.realloc == NULL);
 218         cur_alloc.realloc = static_realloc;
 219         assert(cur_alloc.memalign == NULL);
 220         cur_alloc.memalign = static_memalign;
 221         assert(cur_alloc.posix_memalign == NULL);
 222         cur_alloc.posix_memalign = static_posix_memalign;
 223 }
 224
 225 static
 226 void lookup_all_symbols(void)
 227 {
 228         struct alloc_functions af;
 229
 230         /*
 231          * Temporarily redirect allocation functions to
 232          * static_calloc_aligned, and free function to static_free
 233          * (no-op), until the dlsym lookup has completed.
 234          */
 235         setup_static_allocator();
 236
 237         /* Perform the actual lookups */
 238         af.calloc = dlsym(RTLD_NEXT, "calloc");
 239         af.malloc = dlsym(RTLD_NEXT, "malloc");
 240         af.free = dlsym(RTLD_NEXT, "free");
 241         af.realloc = dlsym(RTLD_NEXT, "realloc");
 242         af.memalign = dlsym(RTLD_NEXT, "memalign");
 243         af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
 244
 245         /* Populate the new allocator functions */
 246         memcpy(&cur_alloc, &af, sizeof(cur_alloc));
 247 }
 248
 249 void *malloc(size_t size)
 250 {
 251         void *retval;
 252
 253         URCU_TLS(malloc_nesting)++;
 254         if (cur_alloc.malloc == NULL) {
 255                 lookup_all_symbols();
 256                 if (cur_alloc.malloc == NULL) {
 257                         fprintf(stderr, "mallocwrap: unable to find malloc\n");
 258                         abort();
 259                 }
 260         }
 261         retval = cur_alloc.malloc(size);
 262         if (URCU_TLS(malloc_nesting) == 1) {
 263                 tracepoint(lttng_ust_libc, malloc,
 264                         size, retval, __builtin_return_address(0));
 265         }
 266         URCU_TLS(malloc_nesting)--;
 267         return retval;
 268 }
 269
 270 void free(void *ptr)
 271 {
 272         URCU_TLS(malloc_nesting)++;
 273         /*
 274          * Check whether the memory was allocated with
 275          * static_calloc_align, in which case there is nothing to free.
 276          */
 277         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 278                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 279                 goto end;
 280         }
 281
 282         if (URCU_TLS(malloc_nesting) == 1) {
 283                 tracepoint(lttng_ust_libc, free,
 284                         ptr, __builtin_return_address(0));
 285         }
 286
 287         if (cur_alloc.free == NULL) {
 288                 lookup_all_symbols();
 289                 if (cur_alloc.free == NULL) {
 290                         fprintf(stderr, "mallocwrap: unable to find free\n");
 291                         abort();
 292                 }
 293         }
 294         cur_alloc.free(ptr);
 295 end:
 296         URCU_TLS(malloc_nesting)--;
 297 }
 298
 299 void *calloc(size_t nmemb, size_t size)
 300 {
 301         void *retval;
 302
 303         URCU_TLS(malloc_nesting)++;
 304         if (cur_alloc.calloc == NULL) {
 305                 lookup_all_symbols();
 306                 if (cur_alloc.calloc == NULL) {
 307                         fprintf(stderr, "callocwrap: unable to find calloc\n");
 308                         abort();
 309                 }
 310         }
 311         retval = cur_alloc.calloc(nmemb, size);
 312         if (URCU_TLS(malloc_nesting) == 1) {
 313                 tracepoint(lttng_ust_libc, calloc,
 314                         nmemb, size, retval, __builtin_return_address(0));
 315         }
 316         URCU_TLS(malloc_nesting)--;
 317         return retval;
 318 }
 319
 320 void *realloc(void *ptr, size_t size)
 321 {
 322         void *retval;
 323
 324         URCU_TLS(malloc_nesting)++;
 325         /*
 326          * Check whether the memory was allocated with
 327          * static_calloc_align, in which case there is nothing
 328          * to free, and we need to copy the old data.
 329          */
 330         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 331                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 332                 size_t *old_size;
 333
 334                 old_size = (size_t *) ptr - 1;
 335                 if (cur_alloc.calloc == NULL) {
 336                         lookup_all_symbols();
 337                         if (cur_alloc.calloc == NULL) {
 338                                 fprintf(stderr, "reallocwrap: unable to find calloc\n");
 339                                 abort();
 340                         }
 341                 }
 342                 retval = cur_alloc.calloc(1, size);
 343                 if (retval) {
 344                         memcpy(retval, ptr, *old_size);
 345                 }
 346                 /*
 347                  * Mimick that a NULL pointer has been received, so
 348                  * memory allocation analysis based on the trace don't
 349                  * get confused by the address from the static
 350                  * allocator.
 351                  */
 352                 ptr = NULL;
 353                 goto end;
 354         }
 355
 356         if (cur_alloc.realloc == NULL) {
 357                 lookup_all_symbols();
 358                 if (cur_alloc.realloc == NULL) {
 359                         fprintf(stderr, "reallocwrap: unable to find realloc\n");
 360                         abort();
 361                 }
 362         }
 363         retval = cur_alloc.realloc(ptr, size);
 364 end:
 365         if (URCU_TLS(malloc_nesting) == 1) {
 366                 tracepoint(lttng_ust_libc, realloc,
 367                         ptr, size, retval, __builtin_return_address(0));
 368         }
 369         URCU_TLS(malloc_nesting)--;
 370         return retval;
 371 }
 372
 373 void *memalign(size_t alignment, size_t size)
 374 {
 375         void *retval;
 376
 377         URCU_TLS(malloc_nesting)++;
 378         if (cur_alloc.memalign == NULL) {
 379                 lookup_all_symbols();
 380                 if (cur_alloc.memalign == NULL) {
 381                         fprintf(stderr, "memalignwrap: unable to find memalign\n");
 382                         abort();
 383                 }
 384         }
 385         retval = cur_alloc.memalign(alignment, size);
 386         if (URCU_TLS(malloc_nesting) == 1) {
 387                 tracepoint(lttng_ust_libc, memalign,
 388                         alignment, size, retval,
 389                         __builtin_return_address(0));
 390         }
 391         URCU_TLS(malloc_nesting)--;
 392         return retval;
 393 }
 394
 395 int posix_memalign(void **memptr, size_t alignment, size_t size)
 396 {
 397         int retval;
 398
 399         URCU_TLS(malloc_nesting)++;
 400         if (cur_alloc.posix_memalign == NULL) {
 401                 lookup_all_symbols();
 402                 if (cur_alloc.posix_memalign == NULL) {
 403                         fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
 404                         abort();
 405                 }
 406         }
 407         retval = cur_alloc.posix_memalign(memptr, alignment, size);
 408         if (URCU_TLS(malloc_nesting) == 1) {
 409                 tracepoint(lttng_ust_libc, posix_memalign,
 410                         *memptr, alignment, size,
 411                         retval, __builtin_return_address(0));
 412         }
 413         URCU_TLS(malloc_nesting)--;
 414         return retval;
 415 }
 416
 417 __attribute__((constructor))
 418 void lttng_ust_malloc_wrapper_init(void)
 419 {
 420         /* Initialization already done */
 421         if (cur_alloc.calloc) {
 422                 return;
 423         }
 424         /*
 425          * Ensure the allocator is in place before the process becomes
 426          * multithreaded.
 427          */
 428         lookup_all_symbols();
 429 }