libcounter/shm.c

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-only
   3  *
   4  * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  */
   6
   7 #define _LGPL_SOURCE
   8 #include <config.h>
   9 #include "shm.h"
  10 #include <unistd.h>
  11 #include <fcntl.h>
  12 #include <sys/mman.h>
  13 #include <sys/types.h>
  14 #include <sys/stat.h>   /* For mode constants */
  15 #include <fcntl.h>      /* For O_* constants */
  16 #include <assert.h>
  17 #include <stdio.h>
  18 #include <signal.h>
  19 #include <dirent.h>
  20 #include <limits.h>
  21 #include <stdbool.h>
  22 #include <stdint.h>
  23
  24 #ifdef HAVE_LIBNUMA
  25 #include <numa.h>
  26 #include <numaif.h>
  27 #endif
  28
  29 #include <lttng/ust-align.h>
  30
  31 #include <ust-helper.h>
  32 #include <ust-fd.h>
  33 #include "../libringbuffer/mmap.h"
  34
  35 /*
  36  * Ensure we have the required amount of space available by writing 0
  37  * into the entire buffer. Not doing so can trigger SIGBUS when going
  38  * beyond the available shm space.
  39  */
  40 static
  41 int zero_file(int fd, size_t len)
  42 {
  43         ssize_t retlen;
  44         size_t written = 0;
  45         char *zeropage;
  46         long pagelen;
  47         int ret;
  48
  49         pagelen = sysconf(_SC_PAGESIZE);
  50         if (pagelen < 0)
  51                 return (int) pagelen;
  52         zeropage = calloc(pagelen, 1);
  53         if (!zeropage)
  54                 return -ENOMEM;
  55
  56         while (len > written) {
  57                 do {
  58                         retlen = write(fd, zeropage,
  59                                 min_t(size_t, pagelen, len - written));
  60                 } while (retlen == -1UL && errno == EINTR);
  61                 if (retlen < 0) {
  62                         ret = (int) retlen;
  63                         goto error;
  64                 }
  65                 written += retlen;
  66         }
  67         ret = 0;
  68 error:
  69         free(zeropage);
  70         return ret;
  71 }
  72
  73 struct lttng_counter_shm_object_table *lttng_counter_shm_object_table_create(size_t max_nb_obj)
  74 {
  75         struct lttng_counter_shm_object_table *table;
  76
  77         table = zmalloc(sizeof(struct lttng_counter_shm_object_table) +
  78                         max_nb_obj * sizeof(table->objects[0]));
  79         if (!table)
  80                 return NULL;
  81         table->size = max_nb_obj;
  82         return table;
  83 }
  84
  85 static
  86 struct lttng_counter_shm_object *_lttng_counter_shm_object_table_alloc_shm(struct lttng_counter_shm_object_table *table,
  87                                            size_t memory_map_size,
  88                                            int cpu_fd)
  89 {
  90         int shmfd, ret;
  91         struct lttng_counter_shm_object *obj;
  92         char *memory_map;
  93
  94         if (cpu_fd < 0)
  95                 return NULL;
  96         if (table->allocated_len >= table->size)
  97                 return NULL;
  98         obj = &table->objects[table->allocated_len];
  99
 100         /* create shm */
 101
 102         shmfd = cpu_fd;
 103         ret = zero_file(shmfd, memory_map_size);
 104         if (ret) {
 105                 PERROR("zero_file");
 106                 goto error_zero_file;
 107         }
 108         ret = ftruncate(shmfd, memory_map_size);
 109         if (ret) {
 110                 PERROR("ftruncate");
 111                 goto error_ftruncate;
 112         }
 113         /*
 114          * Also ensure the file metadata is synced with the storage by using
 115          * fsync(2).
 116          */
 117         ret = fsync(shmfd);
 118         if (ret) {
 119                 PERROR("fsync");
 120                 goto error_fsync;
 121         }
 122         obj->shm_fd_ownership = 0;
 123         obj->shm_fd = shmfd;
 124
 125         /* memory_map: mmap */
 126         memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
 127                           MAP_SHARED | LTTNG_MAP_POPULATE, shmfd, 0);
 128         if (memory_map == MAP_FAILED) {
 129                 PERROR("mmap");
 130                 goto error_mmap;
 131         }
 132         obj->type = LTTNG_COUNTER_SHM_OBJECT_SHM;
 133         obj->memory_map = memory_map;
 134         obj->memory_map_size = memory_map_size;
 135         obj->allocated_len = 0;
 136         obj->index = table->allocated_len++;
 137
 138         return obj;
 139
 140 error_mmap:
 141 error_fsync:
 142 error_ftruncate:
 143 error_zero_file:
 144         return NULL;
 145 }
 146
 147 static
 148 struct lttng_counter_shm_object *_lttng_counter_shm_object_table_alloc_mem(struct lttng_counter_shm_object_table *table,
 149                                            size_t memory_map_size)
 150 {
 151         struct lttng_counter_shm_object *obj;
 152         void *memory_map;
 153
 154         if (table->allocated_len >= table->size)
 155                 return NULL;
 156         obj = &table->objects[table->allocated_len];
 157
 158         memory_map = zmalloc(memory_map_size);
 159         if (!memory_map)
 160                 goto alloc_error;
 161
 162         /* no shm_fd */
 163         obj->shm_fd = -1;
 164         obj->shm_fd_ownership = 0;
 165
 166         obj->type = LTTNG_COUNTER_SHM_OBJECT_MEM;
 167         obj->memory_map = memory_map;
 168         obj->memory_map_size = memory_map_size;
 169         obj->allocated_len = 0;
 170         obj->index = table->allocated_len++;
 171
 172         return obj;
 173
 174 alloc_error:
 175         return NULL;
 176 }
 177
 178 /*
 179  * libnuma prints errors on the console even for numa_available().
 180  * Work-around this limitation by using get_mempolicy() directly to
 181  * check whether the kernel supports mempolicy.
 182  */
 183 #ifdef HAVE_LIBNUMA
 184 static bool lttng_is_numa_available(void)
 185 {
 186         int ret;
 187
 188         ret = get_mempolicy(NULL, NULL, 0, NULL, 0);
 189         if (ret && errno == ENOSYS) {
 190                 return false;
 191         }
 192         return numa_available() > 0;
 193 }
 194 #endif
 195
 196 struct lttng_counter_shm_object *lttng_counter_shm_object_table_alloc(struct lttng_counter_shm_object_table *table,
 197                         size_t memory_map_size,
 198                         enum lttng_counter_shm_object_type type,
 199                         int cpu_fd,
 200                         int cpu)
 201 {
 202         struct lttng_counter_shm_object *shm_object;
 203 #ifdef HAVE_LIBNUMA
 204         int oldnode = 0, node;
 205         bool numa_avail;
 206
 207         numa_avail = lttng_is_numa_available();
 208         if (numa_avail) {
 209                 oldnode = numa_preferred();
 210                 if (cpu >= 0) {
 211                         node = numa_node_of_cpu(cpu);
 212                         if (node >= 0)
 213                                 numa_set_preferred(node);
 214                 }
 215                 if (cpu < 0 || node < 0)
 216                         numa_set_localalloc();
 217         }
 218 #endif /* HAVE_LIBNUMA */
 219         switch (type) {
 220         case LTTNG_COUNTER_SHM_OBJECT_SHM:
 221                 shm_object = _lttng_counter_shm_object_table_alloc_shm(table, memory_map_size,
 222                                 cpu_fd);
 223                 break;
 224         case LTTNG_COUNTER_SHM_OBJECT_MEM:
 225                 shm_object = _lttng_counter_shm_object_table_alloc_mem(table, memory_map_size);
 226                 break;
 227         default:
 228                 assert(0);
 229         }
 230 #ifdef HAVE_LIBNUMA
 231         if (numa_avail)
 232                 numa_set_preferred(oldnode);
 233 #endif /* HAVE_LIBNUMA */
 234         return shm_object;
 235 }
 236
 237 struct lttng_counter_shm_object *lttng_counter_shm_object_table_append_shm(struct lttng_counter_shm_object_table *table,
 238                         int shm_fd,
 239                         size_t memory_map_size)
 240 {
 241         struct lttng_counter_shm_object *obj;
 242         char *memory_map;
 243
 244         if (table->allocated_len >= table->size)
 245                 return NULL;
 246
 247         obj = &table->objects[table->allocated_len];
 248
 249         obj->shm_fd = shm_fd;
 250         obj->shm_fd_ownership = 1;
 251
 252         /* memory_map: mmap */
 253         memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
 254                           MAP_SHARED | LTTNG_MAP_POPULATE, shm_fd, 0);
 255         if (memory_map == MAP_FAILED) {
 256                 PERROR("mmap");
 257                 goto error_mmap;
 258         }
 259         obj->type = LTTNG_COUNTER_SHM_OBJECT_SHM;
 260         obj->memory_map = memory_map;
 261         obj->memory_map_size = memory_map_size;
 262         obj->allocated_len = memory_map_size;
 263         obj->index = table->allocated_len++;
 264
 265         return obj;
 266
 267 error_mmap:
 268         return NULL;
 269 }
 270
 271 /*
 272  * Passing ownership of mem to object.
 273  */
 274 struct lttng_counter_shm_object *lttng_counter_shm_object_table_append_mem(struct lttng_counter_shm_object_table *table,
 275                         void *mem, size_t memory_map_size)
 276 {
 277         struct lttng_counter_shm_object *obj;
 278
 279         if (table->allocated_len >= table->size)
 280                 return NULL;
 281         obj = &table->objects[table->allocated_len];
 282
 283         obj->shm_fd = -1;
 284         obj->shm_fd_ownership = 0;
 285
 286         obj->type = LTTNG_COUNTER_SHM_OBJECT_MEM;
 287         obj->memory_map = mem;
 288         obj->memory_map_size = memory_map_size;
 289         obj->allocated_len = memory_map_size;
 290         obj->index = table->allocated_len++;
 291
 292         return obj;
 293
 294         return NULL;
 295 }
 296
 297 static
 298 void lttng_counter_shmp_object_destroy(struct lttng_counter_shm_object *obj, int consumer)
 299 {
 300         switch (obj->type) {
 301         case LTTNG_COUNTER_SHM_OBJECT_SHM:
 302         {
 303                 int ret;
 304
 305                 ret = munmap(obj->memory_map, obj->memory_map_size);
 306                 if (ret) {
 307                         PERROR("umnmap");
 308                         assert(0);
 309                 }
 310
 311                 if (obj->shm_fd_ownership) {
 312                         /* Delete FDs only if called from app (not consumer). */
 313                         if (!consumer) {
 314                                 lttng_ust_lock_fd_tracker();
 315                                 ret = close(obj->shm_fd);
 316                                 if (!ret) {
 317                                         lttng_ust_delete_fd_from_tracker(obj->shm_fd);
 318                                 } else {
 319                                         PERROR("close");
 320                                         assert(0);
 321                                 }
 322                                 lttng_ust_unlock_fd_tracker();
 323                         } else {
 324                                 ret = close(obj->shm_fd);
 325                                 if (ret) {
 326                                         PERROR("close");
 327                                         assert(0);
 328                                 }
 329                         }
 330                 }
 331                 break;
 332         }
 333         case LTTNG_COUNTER_SHM_OBJECT_MEM:
 334         {
 335                 free(obj->memory_map);
 336                 break;
 337         }
 338         default:
 339                 assert(0);
 340         }
 341 }
 342
 343 void lttng_counter_shm_object_table_destroy(struct lttng_counter_shm_object_table *table, int consumer)
 344 {
 345         int i;
 346
 347         for (i = 0; i < table->allocated_len; i++)
 348                 lttng_counter_shmp_object_destroy(&table->objects[i], consumer);
 349         free(table);
 350 }
 351
 352 /*
 353  * lttng_counter_zalloc_shm - allocate memory within a shm object.
 354  *
 355  * Shared memory is already zeroed by shmget.
 356  * *NOT* multithread-safe (should be protected by mutex).
 357  * Returns a -1, -1 tuple on error.
 358  */
 359 struct lttng_counter_shm_ref lttng_counter_zalloc_shm(struct lttng_counter_shm_object *obj, size_t len)
 360 {
 361         struct lttng_counter_shm_ref ref;
 362         struct lttng_counter_shm_ref shm_ref_error = { -1, -1 };
 363
 364         if (obj->memory_map_size - obj->allocated_len < len)
 365                 return shm_ref_error;
 366         ref.index = obj->index;
 367         ref.offset =  obj->allocated_len;
 368         obj->allocated_len += len;
 369         return ref;
 370 }
 371
 372 void lttng_counter_align_shm(struct lttng_counter_shm_object *obj, size_t align)
 373 {
 374         size_t offset_len = lttng_ust_offset_align(obj->allocated_len, align);
 375         obj->allocated_len += offset_len;
 376 }