liblttng-ust-libc-wrapper/lttng-ust-malloc.c

   1 /*
   2  * Copyright (C) 2009  Pierre-Marc Fournier
   3  * Copyright (C) 2011  Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   4  *
   5  * This library is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation; either
   8  * version 2.1 of the License, or (at your option) any later version.
   9  *
  10  * This library is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with this library; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  18  */
  19
  20 #define _GNU_SOURCE
  21 #define _LGPL_SOURCE
  22 #include <lttng/ust-dlfcn.h>
  23 #include <sys/types.h>
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #include <urcu/system.h>
  27 #include <urcu/uatomic.h>
  28 #include <urcu/compiler.h>
  29 #include <urcu/tls-compat.h>
  30 #include <urcu/arch.h>
  31 #include <lttng/align.h>
  32 #include <helper.h>
  33
  34 #define TRACEPOINT_DEFINE
  35 #define TRACEPOINT_CREATE_PROBES
  36 #define TP_IP_PARAM ip
  37 #include "ust_libc.h"
  38
  39 #define STATIC_CALLOC_LEN 4096
  40 static char static_calloc_buf[STATIC_CALLOC_LEN];
  41 static unsigned long static_calloc_buf_offset;
  42
  43 struct alloc_functions {
  44         void *(*calloc)(size_t nmemb, size_t size);
  45         void *(*malloc)(size_t size);
  46         void (*free)(void *ptr);
  47         void *(*realloc)(void *ptr, size_t size);
  48         void *(*memalign)(size_t alignment, size_t size);
  49         int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
  50 };
  51
  52 static
  53 struct alloc_functions cur_alloc;
  54
  55 /*
  56  * Make sure our own use of the LTS compat layer will not cause infinite
  57  * recursion by calling calloc.
  58  */
  59
  60 static
  61 void *static_calloc(size_t nmemb, size_t size);
  62
  63 /*
  64  * pthread mutex replacement for URCU tls compat layer.
  65  */
  66 static int ust_malloc_lock;
  67
  68 static __attribute__((unused))
  69 void ust_malloc_spin_lock(pthread_mutex_t *lock)
  70 {
  71         /*
  72          * The memory barrier within cmpxchg takes care of ordering
  73          * memory accesses with respect to the start of the critical
  74          * section.
  75          */
  76         while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
  77                 caa_cpu_relax();
  78 }
  79
  80 static __attribute__((unused))
  81 void ust_malloc_spin_unlock(pthread_mutex_t *lock)
  82 {
  83         /*
  84          * Ensure memory accesses within the critical section do not
  85          * leak outside.
  86          */
  87         cmm_smp_mb();
  88         uatomic_set(&ust_malloc_lock, 0);
  89 }
  90
  91 #define calloc static_calloc
  92 #define pthread_mutex_lock ust_malloc_spin_lock
  93 #define pthread_mutex_unlock ust_malloc_spin_unlock
  94 static DEFINE_URCU_TLS(int, malloc_nesting);
  95 #undef ust_malloc_spin_unlock
  96 #undef ust_malloc_spin_lock
  97 #undef calloc
  98
  99 /*
 100  * Static allocator to use when initially executing dlsym(). It keeps a
 101  * size_t value of each object size prior to the object.
 102  */
 103 static
 104 void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
 105 {
 106         size_t prev_offset, new_offset, res_offset, aligned_offset;
 107
 108         if (nmemb * size == 0) {
 109                 return NULL;
 110         }
 111
 112         /*
 113          * Protect static_calloc_buf_offset from concurrent updates
 114          * using a cmpxchg loop rather than a mutex to remove a
 115          * dependency on pthread. This will minimize the risk of bad
 116          * interaction between mutex and malloc instrumentation.
 117          */
 118         res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
 119         do {
 120                 prev_offset = res_offset;
 121                 aligned_offset = ALIGN(prev_offset + sizeof(size_t), alignment);
 122                 new_offset = aligned_offset + nmemb * size;
 123                 if (new_offset > sizeof(static_calloc_buf)) {
 124                         abort();
 125                 }
 126         } while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
 127                         prev_offset, new_offset)) != prev_offset);
 128         *(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
 129         return &static_calloc_buf[aligned_offset];
 130 }
 131
 132 static
 133 void *static_calloc(size_t nmemb, size_t size)
 134 {
 135         void *retval;
 136
 137         retval = static_calloc_aligned(nmemb, size, 1);
 138         return retval;
 139 }
 140
 141 static
 142 void *static_malloc(size_t size)
 143 {
 144         void *retval;
 145
 146         retval = static_calloc_aligned(1, size, 1);
 147         return retval;
 148 }
 149
 150 static
 151 void static_free(void *ptr)
 152 {
 153         /* no-op. */
 154 }
 155
 156 static
 157 void *static_realloc(void *ptr, size_t size)
 158 {
 159         size_t *old_size = NULL;
 160         void *retval;
 161
 162         if (size == 0) {
 163                 retval = NULL;
 164                 goto end;
 165         }
 166
 167         if (ptr) {
 168                 old_size = (size_t *) ptr - 1;
 169                 if (size <= *old_size) {
 170                         /* We can re-use the old entry. */
 171                         *old_size = size;
 172                         retval = ptr;
 173                         goto end;
 174                 }
 175         }
 176         /* We need to expand. Don't free previous memory location. */
 177         retval = static_calloc_aligned(1, size, 1);
 178         assert(retval);
 179         if (ptr)
 180                 memcpy(retval, ptr, *old_size);
 181 end:
 182         return retval;
 183 }
 184
 185 static
 186 void *static_memalign(size_t alignment, size_t size)
 187 {
 188         void *retval;
 189
 190         retval = static_calloc_aligned(1, size, alignment);
 191         return retval;
 192 }
 193
 194 static
 195 int static_posix_memalign(void **memptr, size_t alignment, size_t size)
 196 {
 197         void *ptr;
 198
 199         /* Check for power of 2, larger than void *. */
 200         if (alignment & (alignment - 1)
 201                         || alignment < sizeof(void *)
 202                         || alignment == 0) {
 203                 goto end;
 204         }
 205         ptr = static_calloc_aligned(1, size, alignment);
 206         *memptr = ptr;
 207 end:
 208         return 0;
 209 }
 210
 211 static
 212 void setup_static_allocator(void)
 213 {
 214         assert(cur_alloc.calloc == NULL);
 215         cur_alloc.calloc = static_calloc;
 216         assert(cur_alloc.malloc == NULL);
 217         cur_alloc.malloc = static_malloc;
 218         assert(cur_alloc.free == NULL);
 219         cur_alloc.free = static_free;
 220         assert(cur_alloc.realloc == NULL);
 221         cur_alloc.realloc = static_realloc;
 222         assert(cur_alloc.memalign == NULL);
 223         cur_alloc.memalign = static_memalign;
 224         assert(cur_alloc.posix_memalign == NULL);
 225         cur_alloc.posix_memalign = static_posix_memalign;
 226 }
 227
 228 static
 229 void lookup_all_symbols(void)
 230 {
 231         struct alloc_functions af;
 232
 233         /*
 234          * Temporarily redirect allocation functions to
 235          * static_calloc_aligned, and free function to static_free
 236          * (no-op), until the dlsym lookup has completed.
 237          */
 238         setup_static_allocator();
 239
 240         /* Perform the actual lookups */
 241         af.calloc = dlsym(RTLD_NEXT, "calloc");
 242         af.malloc = dlsym(RTLD_NEXT, "malloc");
 243         af.free = dlsym(RTLD_NEXT, "free");
 244         af.realloc = dlsym(RTLD_NEXT, "realloc");
 245         af.memalign = dlsym(RTLD_NEXT, "memalign");
 246         af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
 247
 248         /* Populate the new allocator functions */
 249         memcpy(&cur_alloc, &af, sizeof(cur_alloc));
 250 }
 251
 252 void *malloc(size_t size)
 253 {
 254         void *retval;
 255
 256         URCU_TLS(malloc_nesting)++;
 257         if (cur_alloc.malloc == NULL) {
 258                 lookup_all_symbols();
 259                 if (cur_alloc.malloc == NULL) {
 260                         fprintf(stderr, "mallocwrap: unable to find malloc\n");
 261                         abort();
 262                 }
 263         }
 264         retval = cur_alloc.malloc(size);
 265         if (URCU_TLS(malloc_nesting) == 1) {
 266                 tracepoint(lttng_ust_libc, malloc,
 267                         size, retval, LTTNG_UST_CALLER_IP());
 268         }
 269         URCU_TLS(malloc_nesting)--;
 270         return retval;
 271 }
 272
 273 void free(void *ptr)
 274 {
 275         URCU_TLS(malloc_nesting)++;
 276         /*
 277          * Check whether the memory was allocated with
 278          * static_calloc_align, in which case there is nothing to free.
 279          */
 280         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 281                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 282                 goto end;
 283         }
 284
 285         if (URCU_TLS(malloc_nesting) == 1) {
 286                 tracepoint(lttng_ust_libc, free,
 287                         ptr, LTTNG_UST_CALLER_IP());
 288         }
 289
 290         if (cur_alloc.free == NULL) {
 291                 lookup_all_symbols();
 292                 if (cur_alloc.free == NULL) {
 293                         fprintf(stderr, "mallocwrap: unable to find free\n");
 294                         abort();
 295                 }
 296         }
 297         cur_alloc.free(ptr);
 298 end:
 299         URCU_TLS(malloc_nesting)--;
 300 }
 301
 302 void *calloc(size_t nmemb, size_t size)
 303 {
 304         void *retval;
 305
 306         URCU_TLS(malloc_nesting)++;
 307         if (cur_alloc.calloc == NULL) {
 308                 lookup_all_symbols();
 309                 if (cur_alloc.calloc == NULL) {
 310                         fprintf(stderr, "callocwrap: unable to find calloc\n");
 311                         abort();
 312                 }
 313         }
 314         retval = cur_alloc.calloc(nmemb, size);
 315         if (URCU_TLS(malloc_nesting) == 1) {
 316                 tracepoint(lttng_ust_libc, calloc,
 317                         nmemb, size, retval, LTTNG_UST_CALLER_IP());
 318         }
 319         URCU_TLS(malloc_nesting)--;
 320         return retval;
 321 }
 322
 323 void *realloc(void *ptr, size_t size)
 324 {
 325         void *retval;
 326
 327         URCU_TLS(malloc_nesting)++;
 328         /*
 329          * Check whether the memory was allocated with
 330          * static_calloc_align, in which case there is nothing
 331          * to free, and we need to copy the old data.
 332          */
 333         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 334                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 335                 size_t *old_size;
 336
 337                 old_size = (size_t *) ptr - 1;
 338                 if (cur_alloc.calloc == NULL) {
 339                         lookup_all_symbols();
 340                         if (cur_alloc.calloc == NULL) {
 341                                 fprintf(stderr, "reallocwrap: unable to find calloc\n");
 342                                 abort();
 343                         }
 344                 }
 345                 retval = cur_alloc.calloc(1, size);
 346                 if (retval) {
 347                         memcpy(retval, ptr, *old_size);
 348                 }
 349                 /*
 350                  * Mimick that a NULL pointer has been received, so
 351                  * memory allocation analysis based on the trace don't
 352                  * get confused by the address from the static
 353                  * allocator.
 354                  */
 355                 ptr = NULL;
 356                 goto end;
 357         }
 358
 359         if (cur_alloc.realloc == NULL) {
 360                 lookup_all_symbols();
 361                 if (cur_alloc.realloc == NULL) {
 362                         fprintf(stderr, "reallocwrap: unable to find realloc\n");
 363                         abort();
 364                 }
 365         }
 366         retval = cur_alloc.realloc(ptr, size);
 367 end:
 368         if (URCU_TLS(malloc_nesting) == 1) {
 369                 tracepoint(lttng_ust_libc, realloc,
 370                         ptr, size, retval, LTTNG_UST_CALLER_IP());
 371         }
 372         URCU_TLS(malloc_nesting)--;
 373         return retval;
 374 }
 375
 376 void *memalign(size_t alignment, size_t size)
 377 {
 378         void *retval;
 379
 380         URCU_TLS(malloc_nesting)++;
 381         if (cur_alloc.memalign == NULL) {
 382                 lookup_all_symbols();
 383                 if (cur_alloc.memalign == NULL) {
 384                         fprintf(stderr, "memalignwrap: unable to find memalign\n");
 385                         abort();
 386                 }
 387         }
 388         retval = cur_alloc.memalign(alignment, size);
 389         if (URCU_TLS(malloc_nesting) == 1) {
 390                 tracepoint(lttng_ust_libc, memalign,
 391                         alignment, size, retval,
 392                         LTTNG_UST_CALLER_IP());
 393         }
 394         URCU_TLS(malloc_nesting)--;
 395         return retval;
 396 }
 397
 398 int posix_memalign(void **memptr, size_t alignment, size_t size)
 399 {
 400         int retval;
 401
 402         URCU_TLS(malloc_nesting)++;
 403         if (cur_alloc.posix_memalign == NULL) {
 404                 lookup_all_symbols();
 405                 if (cur_alloc.posix_memalign == NULL) {
 406                         fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
 407                         abort();
 408                 }
 409         }
 410         retval = cur_alloc.posix_memalign(memptr, alignment, size);
 411         if (URCU_TLS(malloc_nesting) == 1) {
 412                 tracepoint(lttng_ust_libc, posix_memalign,
 413                         *memptr, alignment, size,
 414                         retval, LTTNG_UST_CALLER_IP());
 415         }
 416         URCU_TLS(malloc_nesting)--;
 417         return retval;
 418 }
 419
 420 __attribute__((constructor))
 421 void lttng_ust_malloc_wrapper_init(void)
 422 {
 423         /* Initialization already done */
 424         if (cur_alloc.calloc) {
 425                 return;
 426         }
 427         /*
 428          * Ensure the allocator is in place before the process becomes
 429          * multithreaded.
 430          */
 431         lookup_all_symbols();
 432 }