liblttng-ust-libc-wrapper/lttng-ust-malloc.c

   1 /*
   2  * Copyright (C) 2009  Pierre-Marc Fournier
   3  * Copyright (C) 2011  Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   4  *
   5  * This library is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation; either
   8  * version 2.1 of the License, or (at your option) any later version.
   9  *
  10  * This library is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with this library; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  18  */
  19
  20 #define _GNU_SOURCE
  21 #include <lttng/ust-dlfcn.h>
  22 #include <sys/types.h>
  23 #include <stdio.h>
  24 #include <assert.h>
  25 #include <urcu/system.h>
  26 #include <urcu/uatomic.h>
  27 #include <urcu/compiler.h>
  28 #include <urcu/tls-compat.h>
  29 #include <urcu/arch.h>
  30 #include <lttng/align.h>
  31 #include <helper.h>
  32
  33 #define TRACEPOINT_DEFINE
  34 #define TRACEPOINT_CREATE_PROBES
  35 #define TP_IP_PARAM ip
  36 #include "ust_libc.h"
  37
  38 #define STATIC_CALLOC_LEN 4096
  39 static char static_calloc_buf[STATIC_CALLOC_LEN];
  40 static unsigned long static_calloc_buf_offset;
  41
  42 struct alloc_functions {
  43         void *(*calloc)(size_t nmemb, size_t size);
  44         void *(*malloc)(size_t size);
  45         void (*free)(void *ptr);
  46         void *(*realloc)(void *ptr, size_t size);
  47         void *(*memalign)(size_t alignment, size_t size);
  48         int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
  49 };
  50
  51 static
  52 struct alloc_functions cur_alloc;
  53
  54 /*
  55  * Make sure our own use of the LTS compat layer will not cause infinite
  56  * recursion by calling calloc.
  57  */
  58
  59 static
  60 void *static_calloc(size_t nmemb, size_t size);
  61
  62 /*
  63  * pthread mutex replacement for URCU tls compat layer.
  64  */
  65 static int ust_malloc_lock;
  66
  67 static __attribute__((unused))
  68 void ust_malloc_spin_lock(pthread_mutex_t *lock)
  69 {
  70         /*
  71          * The memory barrier within cmpxchg takes care of ordering
  72          * memory accesses with respect to the start of the critical
  73          * section.
  74          */
  75         while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
  76                 caa_cpu_relax();
  77 }
  78
  79 static __attribute__((unused))
  80 void ust_malloc_spin_unlock(pthread_mutex_t *lock)
  81 {
  82         /*
  83          * Ensure memory accesses within the critical section do not
  84          * leak outside.
  85          */
  86         cmm_smp_mb();
  87         uatomic_set(&ust_malloc_lock, 0);
  88 }
  89
  90 #define calloc static_calloc
  91 #define pthread_mutex_lock ust_malloc_spin_lock
  92 #define pthread_mutex_unlock ust_malloc_spin_unlock
  93 static DEFINE_URCU_TLS(int, malloc_nesting);
  94 #undef ust_malloc_spin_unlock
  95 #undef ust_malloc_spin_lock
  96 #undef calloc
  97
  98 /*
  99  * Static allocator to use when initially executing dlsym(). It keeps a
 100  * size_t value of each object size prior to the object.
 101  */
 102 static
 103 void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
 104 {
 105         size_t prev_offset, new_offset, res_offset, aligned_offset;
 106
 107         if (nmemb * size == 0) {
 108                 return NULL;
 109         }
 110
 111         /*
 112          * Protect static_calloc_buf_offset from concurrent updates
 113          * using a cmpxchg loop rather than a mutex to remove a
 114          * dependency on pthread. This will minimize the risk of bad
 115          * interaction between mutex and malloc instrumentation.
 116          */
 117         res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
 118         do {
 119                 prev_offset = res_offset;
 120                 aligned_offset = ALIGN(prev_offset + sizeof(size_t), alignment);
 121                 new_offset = aligned_offset + nmemb * size;
 122                 if (new_offset > sizeof(static_calloc_buf)) {
 123                         abort();
 124                 }
 125         } while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
 126                         prev_offset, new_offset)) != prev_offset);
 127         *(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
 128         return &static_calloc_buf[aligned_offset];
 129 }
 130
 131 static
 132 void *static_calloc(size_t nmemb, size_t size)
 133 {
 134         void *retval;
 135
 136         retval = static_calloc_aligned(nmemb, size, 1);
 137         return retval;
 138 }
 139
 140 static
 141 void *static_malloc(size_t size)
 142 {
 143         void *retval;
 144
 145         retval = static_calloc_aligned(1, size, 1);
 146         return retval;
 147 }
 148
 149 static
 150 void static_free(void *ptr)
 151 {
 152         /* no-op. */
 153 }
 154
 155 static
 156 void *static_realloc(void *ptr, size_t size)
 157 {
 158         size_t *old_size = NULL;
 159         void *retval;
 160
 161         if (size == 0) {
 162                 retval = NULL;
 163                 goto end;
 164         }
 165
 166         if (ptr) {
 167                 old_size = (size_t *) ptr - 1;
 168                 if (size <= *old_size) {
 169                         /* We can re-use the old entry. */
 170                         *old_size = size;
 171                         retval = ptr;
 172                         goto end;
 173                 }
 174         }
 175         /* We need to expand. Don't free previous memory location. */
 176         retval = static_calloc_aligned(1, size, 1);
 177         assert(retval);
 178         if (ptr)
 179                 memcpy(retval, ptr, *old_size);
 180 end:
 181         return retval;
 182 }
 183
 184 static
 185 void *static_memalign(size_t alignment, size_t size)
 186 {
 187         void *retval;
 188
 189         retval = static_calloc_aligned(1, size, alignment);
 190         return retval;
 191 }
 192
 193 static
 194 int static_posix_memalign(void **memptr, size_t alignment, size_t size)
 195 {
 196         void *ptr;
 197
 198         /* Check for power of 2, larger than void *. */
 199         if (alignment & (alignment - 1)
 200                         || alignment < sizeof(void *)
 201                         || alignment == 0) {
 202                 goto end;
 203         }
 204         ptr = static_calloc_aligned(1, size, alignment);
 205         *memptr = ptr;
 206 end:
 207         return 0;
 208 }
 209
 210 static
 211 void setup_static_allocator(void)
 212 {
 213         assert(cur_alloc.calloc == NULL);
 214         cur_alloc.calloc = static_calloc;
 215         assert(cur_alloc.malloc == NULL);
 216         cur_alloc.malloc = static_malloc;
 217         assert(cur_alloc.free == NULL);
 218         cur_alloc.free = static_free;
 219         assert(cur_alloc.realloc == NULL);
 220         cur_alloc.realloc = static_realloc;
 221         assert(cur_alloc.memalign == NULL);
 222         cur_alloc.memalign = static_memalign;
 223         assert(cur_alloc.posix_memalign == NULL);
 224         cur_alloc.posix_memalign = static_posix_memalign;
 225 }
 226
 227 static
 228 void lookup_all_symbols(void)
 229 {
 230         struct alloc_functions af;
 231
 232         /*
 233          * Temporarily redirect allocation functions to
 234          * static_calloc_aligned, and free function to static_free
 235          * (no-op), until the dlsym lookup has completed.
 236          */
 237         setup_static_allocator();
 238
 239         /* Perform the actual lookups */
 240         af.calloc = dlsym(RTLD_NEXT, "calloc");
 241         af.malloc = dlsym(RTLD_NEXT, "malloc");
 242         af.free = dlsym(RTLD_NEXT, "free");
 243         af.realloc = dlsym(RTLD_NEXT, "realloc");
 244         af.memalign = dlsym(RTLD_NEXT, "memalign");
 245         af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
 246
 247         /* Populate the new allocator functions */
 248         memcpy(&cur_alloc, &af, sizeof(cur_alloc));
 249 }
 250
 251 void *malloc(size_t size)
 252 {
 253         void *retval;
 254
 255         URCU_TLS(malloc_nesting)++;
 256         if (cur_alloc.malloc == NULL) {
 257                 lookup_all_symbols();
 258                 if (cur_alloc.malloc == NULL) {
 259                         fprintf(stderr, "mallocwrap: unable to find malloc\n");
 260                         abort();
 261                 }
 262         }
 263         retval = cur_alloc.malloc(size);
 264         if (URCU_TLS(malloc_nesting) == 1) {
 265                 tracepoint(lttng_ust_libc, malloc,
 266                         size, retval, LTTNG_UST_CALLER_IP());
 267         }
 268         URCU_TLS(malloc_nesting)--;
 269         return retval;
 270 }
 271
 272 void free(void *ptr)
 273 {
 274         URCU_TLS(malloc_nesting)++;
 275         /*
 276          * Check whether the memory was allocated with
 277          * static_calloc_align, in which case there is nothing to free.
 278          */
 279         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 280                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 281                 goto end;
 282         }
 283
 284         if (URCU_TLS(malloc_nesting) == 1) {
 285                 tracepoint(lttng_ust_libc, free,
 286                         ptr, LTTNG_UST_CALLER_IP());
 287         }
 288
 289         if (cur_alloc.free == NULL) {
 290                 lookup_all_symbols();
 291                 if (cur_alloc.free == NULL) {
 292                         fprintf(stderr, "mallocwrap: unable to find free\n");
 293                         abort();
 294                 }
 295         }
 296         cur_alloc.free(ptr);
 297 end:
 298         URCU_TLS(malloc_nesting)--;
 299 }
 300
 301 void *calloc(size_t nmemb, size_t size)
 302 {
 303         void *retval;
 304
 305         URCU_TLS(malloc_nesting)++;
 306         if (cur_alloc.calloc == NULL) {
 307                 lookup_all_symbols();
 308                 if (cur_alloc.calloc == NULL) {
 309                         fprintf(stderr, "callocwrap: unable to find calloc\n");
 310                         abort();
 311                 }
 312         }
 313         retval = cur_alloc.calloc(nmemb, size);
 314         if (URCU_TLS(malloc_nesting) == 1) {
 315                 tracepoint(lttng_ust_libc, calloc,
 316                         nmemb, size, retval, LTTNG_UST_CALLER_IP());
 317         }
 318         URCU_TLS(malloc_nesting)--;
 319         return retval;
 320 }
 321
 322 void *realloc(void *ptr, size_t size)
 323 {
 324         void *retval;
 325
 326         URCU_TLS(malloc_nesting)++;
 327         /*
 328          * Check whether the memory was allocated with
 329          * static_calloc_align, in which case there is nothing
 330          * to free, and we need to copy the old data.
 331          */
 332         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 333                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 334                 size_t *old_size;
 335
 336                 old_size = (size_t *) ptr - 1;
 337                 if (cur_alloc.calloc == NULL) {
 338                         lookup_all_symbols();
 339                         if (cur_alloc.calloc == NULL) {
 340                                 fprintf(stderr, "reallocwrap: unable to find calloc\n");
 341                                 abort();
 342                         }
 343                 }
 344                 retval = cur_alloc.calloc(1, size);
 345                 if (retval) {
 346                         memcpy(retval, ptr, *old_size);
 347                 }
 348                 /*
 349                  * Mimick that a NULL pointer has been received, so
 350                  * memory allocation analysis based on the trace don't
 351                  * get confused by the address from the static
 352                  * allocator.
 353                  */
 354                 ptr = NULL;
 355                 goto end;
 356         }
 357
 358         if (cur_alloc.realloc == NULL) {
 359                 lookup_all_symbols();
 360                 if (cur_alloc.realloc == NULL) {
 361                         fprintf(stderr, "reallocwrap: unable to find realloc\n");
 362                         abort();
 363                 }
 364         }
 365         retval = cur_alloc.realloc(ptr, size);
 366 end:
 367         if (URCU_TLS(malloc_nesting) == 1) {
 368                 tracepoint(lttng_ust_libc, realloc,
 369                         ptr, size, retval, LTTNG_UST_CALLER_IP());
 370         }
 371         URCU_TLS(malloc_nesting)--;
 372         return retval;
 373 }
 374
 375 void *memalign(size_t alignment, size_t size)
 376 {
 377         void *retval;
 378
 379         URCU_TLS(malloc_nesting)++;
 380         if (cur_alloc.memalign == NULL) {
 381                 lookup_all_symbols();
 382                 if (cur_alloc.memalign == NULL) {
 383                         fprintf(stderr, "memalignwrap: unable to find memalign\n");
 384                         abort();
 385                 }
 386         }
 387         retval = cur_alloc.memalign(alignment, size);
 388         if (URCU_TLS(malloc_nesting) == 1) {
 389                 tracepoint(lttng_ust_libc, memalign,
 390                         alignment, size, retval,
 391                         LTTNG_UST_CALLER_IP());
 392         }
 393         URCU_TLS(malloc_nesting)--;
 394         return retval;
 395 }
 396
 397 int posix_memalign(void **memptr, size_t alignment, size_t size)
 398 {
 399         int retval;
 400
 401         URCU_TLS(malloc_nesting)++;
 402         if (cur_alloc.posix_memalign == NULL) {
 403                 lookup_all_symbols();
 404                 if (cur_alloc.posix_memalign == NULL) {
 405                         fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
 406                         abort();
 407                 }
 408         }
 409         retval = cur_alloc.posix_memalign(memptr, alignment, size);
 410         if (URCU_TLS(malloc_nesting) == 1) {
 411                 tracepoint(lttng_ust_libc, posix_memalign,
 412                         *memptr, alignment, size,
 413                         retval, LTTNG_UST_CALLER_IP());
 414         }
 415         URCU_TLS(malloc_nesting)--;
 416         return retval;
 417 }
 418
 419 __attribute__((constructor))
 420 void lttng_ust_malloc_wrapper_init(void)
 421 {
 422         /* Initialization already done */
 423         if (cur_alloc.calloc) {
 424                 return;
 425         }
 426         /*
 427          * Ensure the allocator is in place before the process becomes
 428          * multithreaded.
 429          */
 430         lookup_all_symbols();
 431 }