liblttng-ust-libc-wrapper/lttng-ust-malloc.c

   1 /*
   2  * Copyright (C) 2009  Pierre-Marc Fournier
   3  * Copyright (C) 2011  Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   4  *
   5  * This library is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation; either
   8  * version 2.1 of the License, or (at your option) any later version.
   9  *
  10  * This library is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with this library; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  18  */
  19
  20 /*
  21  * Do _not_ define _LGPL_SOURCE because we don't want to create a
  22  * circular dependency loop between this malloc wrapper, liburcu and
  23  * libc.
  24  */
  25 #include <lttng/ust-dlfcn.h>
  26 #include <sys/types.h>
  27 #include <stdio.h>
  28 #include <assert.h>
  29 #include <urcu/system.h>
  30 #include <urcu/uatomic.h>
  31 #include <urcu/compiler.h>
  32 #include <urcu/tls-compat.h>
  33 #include <urcu/arch.h>
  34 #include <lttng/align.h>
  35 #include <helper.h>
  36
  37 #define TRACEPOINT_DEFINE
  38 #define TRACEPOINT_CREATE_PROBES
  39 #define TP_IP_PARAM ip
  40 #include "ust_libc.h"
  41
  42 #define STATIC_CALLOC_LEN 4096
  43 static char static_calloc_buf[STATIC_CALLOC_LEN];
  44 static unsigned long static_calloc_buf_offset;
  45
  46 struct alloc_functions {
  47         void *(*calloc)(size_t nmemb, size_t size);
  48         void *(*malloc)(size_t size);
  49         void (*free)(void *ptr);
  50         void *(*realloc)(void *ptr, size_t size);
  51         void *(*memalign)(size_t alignment, size_t size);
  52         int (*posix_memalign)(void **memptr, size_t alignment, size_t size);
  53 };
  54
  55 static
  56 struct alloc_functions cur_alloc;
  57
  58 /*
  59  * Make sure our own use of the LTS compat layer will not cause infinite
  60  * recursion by calling calloc.
  61  */
  62
  63 static
  64 void *static_calloc(size_t nmemb, size_t size);
  65
  66 /*
  67  * pthread mutex replacement for URCU tls compat layer.
  68  */
  69 static int ust_malloc_lock;
  70
  71 static __attribute__((unused))
  72 void ust_malloc_spin_lock(pthread_mutex_t *lock)
  73 {
  74         /*
  75          * The memory barrier within cmpxchg takes care of ordering
  76          * memory accesses with respect to the start of the critical
  77          * section.
  78          */
  79         while (uatomic_cmpxchg(&ust_malloc_lock, 0, 1) != 0)
  80                 caa_cpu_relax();
  81 }
  82
  83 static __attribute__((unused))
  84 void ust_malloc_spin_unlock(pthread_mutex_t *lock)
  85 {
  86         /*
  87          * Ensure memory accesses within the critical section do not
  88          * leak outside.
  89          */
  90         cmm_smp_mb();
  91         uatomic_set(&ust_malloc_lock, 0);
  92 }
  93
  94 #define calloc static_calloc
  95 #define pthread_mutex_lock ust_malloc_spin_lock
  96 #define pthread_mutex_unlock ust_malloc_spin_unlock
  97 static DEFINE_URCU_TLS(int, malloc_nesting);
  98 #undef pthread_mutex_unlock
  99 #undef pthread_mutex_lock
 100 #undef calloc
 101
 102 /*
 103  * Static allocator to use when initially executing dlsym(). It keeps a
 104  * size_t value of each object size prior to the object.
 105  */
 106 static
 107 void *static_calloc_aligned(size_t nmemb, size_t size, size_t alignment)
 108 {
 109         size_t prev_offset, new_offset, res_offset, aligned_offset;
 110
 111         if (nmemb * size == 0) {
 112                 return NULL;
 113         }
 114
 115         /*
 116          * Protect static_calloc_buf_offset from concurrent updates
 117          * using a cmpxchg loop rather than a mutex to remove a
 118          * dependency on pthread. This will minimize the risk of bad
 119          * interaction between mutex and malloc instrumentation.
 120          */
 121         res_offset = CMM_LOAD_SHARED(static_calloc_buf_offset);
 122         do {
 123                 prev_offset = res_offset;
 124                 aligned_offset = LTTNG_UST_ALIGN(prev_offset + sizeof(size_t), alignment);
 125                 new_offset = aligned_offset + nmemb * size;
 126                 if (new_offset > sizeof(static_calloc_buf)) {
 127                         abort();
 128                 }
 129         } while ((res_offset = uatomic_cmpxchg(&static_calloc_buf_offset,
 130                         prev_offset, new_offset)) != prev_offset);
 131         *(size_t *) &static_calloc_buf[aligned_offset - sizeof(size_t)] = size;
 132         return &static_calloc_buf[aligned_offset];
 133 }
 134
 135 static
 136 void *static_calloc(size_t nmemb, size_t size)
 137 {
 138         void *retval;
 139
 140         retval = static_calloc_aligned(nmemb, size, 1);
 141         return retval;
 142 }
 143
 144 static
 145 void *static_malloc(size_t size)
 146 {
 147         void *retval;
 148
 149         retval = static_calloc_aligned(1, size, 1);
 150         return retval;
 151 }
 152
 153 static
 154 void static_free(void *ptr)
 155 {
 156         /* no-op. */
 157 }
 158
 159 static
 160 void *static_realloc(void *ptr, size_t size)
 161 {
 162         size_t *old_size = NULL;
 163         void *retval;
 164
 165         if (size == 0) {
 166                 retval = NULL;
 167                 goto end;
 168         }
 169
 170         if (ptr) {
 171                 old_size = (size_t *) ptr - 1;
 172                 if (size <= *old_size) {
 173                         /* We can re-use the old entry. */
 174                         *old_size = size;
 175                         retval = ptr;
 176                         goto end;
 177                 }
 178         }
 179         /* We need to expand. Don't free previous memory location. */
 180         retval = static_calloc_aligned(1, size, 1);
 181         assert(retval);
 182         if (ptr)
 183                 memcpy(retval, ptr, *old_size);
 184 end:
 185         return retval;
 186 }
 187
 188 static
 189 void *static_memalign(size_t alignment, size_t size)
 190 {
 191         void *retval;
 192
 193         retval = static_calloc_aligned(1, size, alignment);
 194         return retval;
 195 }
 196
 197 static
 198 int static_posix_memalign(void **memptr, size_t alignment, size_t size)
 199 {
 200         void *ptr;
 201
 202         /* Check for power of 2, larger than void *. */
 203         if (alignment & (alignment - 1)
 204                         || alignment < sizeof(void *)
 205                         || alignment == 0) {
 206                 goto end;
 207         }
 208         ptr = static_calloc_aligned(1, size, alignment);
 209         *memptr = ptr;
 210 end:
 211         return 0;
 212 }
 213
 214 static
 215 void setup_static_allocator(void)
 216 {
 217         assert(cur_alloc.calloc == NULL);
 218         cur_alloc.calloc = static_calloc;
 219         assert(cur_alloc.malloc == NULL);
 220         cur_alloc.malloc = static_malloc;
 221         assert(cur_alloc.free == NULL);
 222         cur_alloc.free = static_free;
 223         assert(cur_alloc.realloc == NULL);
 224         cur_alloc.realloc = static_realloc;
 225         assert(cur_alloc.memalign == NULL);
 226         cur_alloc.memalign = static_memalign;
 227         assert(cur_alloc.posix_memalign == NULL);
 228         cur_alloc.posix_memalign = static_posix_memalign;
 229 }
 230
 231 static
 232 void lookup_all_symbols(void)
 233 {
 234         struct alloc_functions af;
 235
 236         /*
 237          * Temporarily redirect allocation functions to
 238          * static_calloc_aligned, and free function to static_free
 239          * (no-op), until the dlsym lookup has completed.
 240          */
 241         setup_static_allocator();
 242
 243         /* Perform the actual lookups */
 244         af.calloc = dlsym(RTLD_NEXT, "calloc");
 245         af.malloc = dlsym(RTLD_NEXT, "malloc");
 246         af.free = dlsym(RTLD_NEXT, "free");
 247         af.realloc = dlsym(RTLD_NEXT, "realloc");
 248         af.memalign = dlsym(RTLD_NEXT, "memalign");
 249         af.posix_memalign = dlsym(RTLD_NEXT, "posix_memalign");
 250
 251         /* Populate the new allocator functions */
 252         memcpy(&cur_alloc, &af, sizeof(cur_alloc));
 253 }
 254
 255 void *malloc(size_t size)
 256 {
 257         void *retval;
 258
 259         URCU_TLS(malloc_nesting)++;
 260         if (cur_alloc.malloc == NULL) {
 261                 lookup_all_symbols();
 262                 if (cur_alloc.malloc == NULL) {
 263                         fprintf(stderr, "mallocwrap: unable to find malloc\n");
 264                         abort();
 265                 }
 266         }
 267         retval = cur_alloc.malloc(size);
 268         if (URCU_TLS(malloc_nesting) == 1) {
 269                 tracepoint(lttng_ust_libc, malloc,
 270                         size, retval, LTTNG_UST_CALLER_IP());
 271         }
 272         URCU_TLS(malloc_nesting)--;
 273         return retval;
 274 }
 275
 276 void free(void *ptr)
 277 {
 278         URCU_TLS(malloc_nesting)++;
 279         /*
 280          * Check whether the memory was allocated with
 281          * static_calloc_align, in which case there is nothing to free.
 282          */
 283         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 284                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 285                 goto end;
 286         }
 287
 288         if (URCU_TLS(malloc_nesting) == 1) {
 289                 tracepoint(lttng_ust_libc, free,
 290                         ptr, LTTNG_UST_CALLER_IP());
 291         }
 292
 293         if (cur_alloc.free == NULL) {
 294                 lookup_all_symbols();
 295                 if (cur_alloc.free == NULL) {
 296                         fprintf(stderr, "mallocwrap: unable to find free\n");
 297                         abort();
 298                 }
 299         }
 300         cur_alloc.free(ptr);
 301 end:
 302         URCU_TLS(malloc_nesting)--;
 303 }
 304
 305 void *calloc(size_t nmemb, size_t size)
 306 {
 307         void *retval;
 308
 309         URCU_TLS(malloc_nesting)++;
 310         if (cur_alloc.calloc == NULL) {
 311                 lookup_all_symbols();
 312                 if (cur_alloc.calloc == NULL) {
 313                         fprintf(stderr, "callocwrap: unable to find calloc\n");
 314                         abort();
 315                 }
 316         }
 317         retval = cur_alloc.calloc(nmemb, size);
 318         if (URCU_TLS(malloc_nesting) == 1) {
 319                 tracepoint(lttng_ust_libc, calloc,
 320                         nmemb, size, retval, LTTNG_UST_CALLER_IP());
 321         }
 322         URCU_TLS(malloc_nesting)--;
 323         return retval;
 324 }
 325
 326 void *realloc(void *ptr, size_t size)
 327 {
 328         void *retval;
 329
 330         URCU_TLS(malloc_nesting)++;
 331         /*
 332          * Check whether the memory was allocated with
 333          * static_calloc_align, in which case there is nothing
 334          * to free, and we need to copy the old data.
 335          */
 336         if (caa_unlikely((char *)ptr >= static_calloc_buf &&
 337                         (char *)ptr < static_calloc_buf + STATIC_CALLOC_LEN)) {
 338                 size_t *old_size;
 339
 340                 old_size = (size_t *) ptr - 1;
 341                 if (cur_alloc.calloc == NULL) {
 342                         lookup_all_symbols();
 343                         if (cur_alloc.calloc == NULL) {
 344                                 fprintf(stderr, "reallocwrap: unable to find calloc\n");
 345                                 abort();
 346                         }
 347                 }
 348                 retval = cur_alloc.calloc(1, size);
 349                 if (retval) {
 350                         memcpy(retval, ptr, *old_size);
 351                 }
 352                 /*
 353                  * Mimick that a NULL pointer has been received, so
 354                  * memory allocation analysis based on the trace don't
 355                  * get confused by the address from the static
 356                  * allocator.
 357                  */
 358                 ptr = NULL;
 359                 goto end;
 360         }
 361
 362         if (cur_alloc.realloc == NULL) {
 363                 lookup_all_symbols();
 364                 if (cur_alloc.realloc == NULL) {
 365                         fprintf(stderr, "reallocwrap: unable to find realloc\n");
 366                         abort();
 367                 }
 368         }
 369         retval = cur_alloc.realloc(ptr, size);
 370 end:
 371         if (URCU_TLS(malloc_nesting) == 1) {
 372                 tracepoint(lttng_ust_libc, realloc,
 373                         ptr, size, retval, LTTNG_UST_CALLER_IP());
 374         }
 375         URCU_TLS(malloc_nesting)--;
 376         return retval;
 377 }
 378
 379 void *memalign(size_t alignment, size_t size)
 380 {
 381         void *retval;
 382
 383         URCU_TLS(malloc_nesting)++;
 384         if (cur_alloc.memalign == NULL) {
 385                 lookup_all_symbols();
 386                 if (cur_alloc.memalign == NULL) {
 387                         fprintf(stderr, "memalignwrap: unable to find memalign\n");
 388                         abort();
 389                 }
 390         }
 391         retval = cur_alloc.memalign(alignment, size);
 392         if (URCU_TLS(malloc_nesting) == 1) {
 393                 tracepoint(lttng_ust_libc, memalign,
 394                         alignment, size, retval,
 395                         LTTNG_UST_CALLER_IP());
 396         }
 397         URCU_TLS(malloc_nesting)--;
 398         return retval;
 399 }
 400
 401 int posix_memalign(void **memptr, size_t alignment, size_t size)
 402 {
 403         int retval;
 404
 405         URCU_TLS(malloc_nesting)++;
 406         if (cur_alloc.posix_memalign == NULL) {
 407                 lookup_all_symbols();
 408                 if (cur_alloc.posix_memalign == NULL) {
 409                         fprintf(stderr, "posix_memalignwrap: unable to find posix_memalign\n");
 410                         abort();
 411                 }
 412         }
 413         retval = cur_alloc.posix_memalign(memptr, alignment, size);
 414         if (URCU_TLS(malloc_nesting) == 1) {
 415                 tracepoint(lttng_ust_libc, posix_memalign,
 416                         *memptr, alignment, size,
 417                         retval, LTTNG_UST_CALLER_IP());
 418         }
 419         URCU_TLS(malloc_nesting)--;
 420         return retval;
 421 }
 422
 423 static
 424 void lttng_ust_fixup_malloc_nesting_tls(void)
 425 {
 426         asm volatile ("" : : "m" (URCU_TLS(malloc_nesting)));
 427 }
 428
 429 __attribute__((constructor))
 430 void lttng_ust_malloc_wrapper_init(void)
 431 {
 432         /* Initialization already done */
 433         if (cur_alloc.calloc) {
 434                 return;
 435         }
 436         lttng_ust_fixup_malloc_nesting_tls();
 437         /*
 438          * Ensure the allocator is in place before the process becomes
 439          * multithreaded.
 440          */
 441         lookup_all_symbols();
 442 }