libringbuffer/shm.c

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-only
   3  *
   4  * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  */
   6
   7 #define _LGPL_SOURCE
   8 #include "shm.h"
   9 #include <unistd.h>
  10 #include <fcntl.h>
  11 #include <sys/mman.h>
  12 #include <sys/types.h>
  13 #include <sys/stat.h>   /* For mode constants */
  14 #include <fcntl.h>      /* For O_* constants */
  15 #include <assert.h>
  16 #include <stdio.h>
  17 #include <signal.h>
  18 #include <dirent.h>
  19 #include <lttng/align.h>
  20 #include <limits.h>
  21 #include <stdbool.h>
  22 #include <stdint.h>
  23 #ifdef HAVE_LIBNUMA
  24 #include <numa.h>
  25 #include <numaif.h>
  26 #endif
  27 #include <helper.h>
  28 #include <ust-fd.h>
  29 #include "mmap.h"
  30
  31 /*
  32  * Ensure we have the required amount of space available by writing 0
  33  * into the entire buffer. Not doing so can trigger SIGBUS when going
  34  * beyond the available shm space.
  35  */
  36 static
  37 int zero_file(int fd, size_t len)
  38 {
  39         ssize_t retlen;
  40         size_t written = 0;
  41         char *zeropage;
  42         long pagelen;
  43         int ret;
  44
  45         pagelen = sysconf(_SC_PAGESIZE);
  46         if (pagelen < 0)
  47                 return (int) pagelen;
  48         zeropage = calloc(pagelen, 1);
  49         if (!zeropage)
  50                 return -ENOMEM;
  51
  52         while (len > written) {
  53                 do {
  54                         retlen = write(fd, zeropage,
  55                                 min_t(size_t, pagelen, len - written));
  56                 } while (retlen == -1UL && errno == EINTR);
  57                 if (retlen < 0) {
  58                         ret = (int) retlen;
  59                         goto error;
  60                 }
  61                 written += retlen;
  62         }
  63         ret = 0;
  64 error:
  65         free(zeropage);
  66         return ret;
  67 }
  68
  69 struct shm_object_table *shm_object_table_create(size_t max_nb_obj)
  70 {
  71         struct shm_object_table *table;
  72
  73         table = zmalloc(sizeof(struct shm_object_table) +
  74                         max_nb_obj * sizeof(table->objects[0]));
  75         if (!table)
  76                 return NULL;
  77         table->size = max_nb_obj;
  78         return table;
  79 }
  80
  81 static
  82 struct shm_object *_shm_object_table_alloc_shm(struct shm_object_table *table,
  83                                            size_t memory_map_size,
  84                                            int stream_fd)
  85 {
  86         int shmfd, waitfd[2], ret, i;
  87         struct shm_object *obj;
  88         char *memory_map;
  89
  90         if (stream_fd < 0)
  91                 return NULL;
  92         if (table->allocated_len >= table->size)
  93                 return NULL;
  94         obj = &table->objects[table->allocated_len];
  95
  96         /* wait_fd: create pipe */
  97         ret = pipe(waitfd);
  98         if (ret < 0) {
  99                 PERROR("pipe");
 100                 goto error_pipe;
 101         }
 102         for (i = 0; i < 2; i++) {
 103                 ret = fcntl(waitfd[i], F_SETFD, FD_CLOEXEC);
 104                 if (ret < 0) {
 105                         PERROR("fcntl");
 106                         goto error_fcntl;
 107                 }
 108         }
 109         /* The write end of the pipe needs to be non-blocking */
 110         ret = fcntl(waitfd[1], F_SETFL, O_NONBLOCK);
 111         if (ret < 0) {
 112                 PERROR("fcntl");
 113                 goto error_fcntl;
 114         }
 115         memcpy(obj->wait_fd, waitfd, sizeof(waitfd));
 116
 117         /*
 118          * Set POSIX shared memory object size
 119          *
 120          * First, use ftruncate() to set its size, some implementations won't
 121          * allow writes past the size set by ftruncate.
 122          * Then, use write() to fill it with zeros, this allows us to fully
 123          * allocate it and detect a shortage of shm space without dealing with
 124          * a SIGBUS.
 125          */
 126
 127         shmfd = stream_fd;
 128         ret = ftruncate(shmfd, memory_map_size);
 129         if (ret) {
 130                 PERROR("ftruncate");
 131                 goto error_ftruncate;
 132         }
 133         ret = zero_file(shmfd, memory_map_size);
 134         if (ret) {
 135                 PERROR("zero_file");
 136                 goto error_zero_file;
 137         }
 138
 139         /*
 140          * Also ensure the file metadata is synced with the storage by using
 141          * fsync(2). Some platforms don't allow fsync on POSIX shm fds, ignore
 142          * EINVAL accordingly.
 143          */
 144         ret = fsync(shmfd);
 145         if (ret && errno != EINVAL) {
 146                 PERROR("fsync");
 147                 goto error_fsync;
 148         }
 149         obj->shm_fd_ownership = 0;
 150         obj->shm_fd = shmfd;
 151
 152         /* memory_map: mmap */
 153         memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
 154                           MAP_SHARED | LTTNG_MAP_POPULATE, shmfd, 0);
 155         if (memory_map == MAP_FAILED) {
 156                 PERROR("mmap");
 157                 goto error_mmap;
 158         }
 159         obj->type = SHM_OBJECT_SHM;
 160         obj->memory_map = memory_map;
 161         obj->memory_map_size = memory_map_size;
 162         obj->allocated_len = 0;
 163         obj->index = table->allocated_len++;
 164
 165         return obj;
 166
 167 error_mmap:
 168 error_fsync:
 169 error_ftruncate:
 170 error_zero_file:
 171 error_fcntl:
 172         for (i = 0; i < 2; i++) {
 173                 ret = close(waitfd[i]);
 174                 if (ret) {
 175                         PERROR("close");
 176                         assert(0);
 177                 }
 178         }
 179 error_pipe:
 180         return NULL;
 181 }
 182
 183 static
 184 struct shm_object *_shm_object_table_alloc_mem(struct shm_object_table *table,
 185                                            size_t memory_map_size)
 186 {
 187         struct shm_object *obj;
 188         void *memory_map;
 189         int waitfd[2], i, ret;
 190
 191         if (table->allocated_len >= table->size)
 192                 return NULL;
 193         obj = &table->objects[table->allocated_len];
 194
 195         memory_map = zmalloc(memory_map_size);
 196         if (!memory_map)
 197                 goto alloc_error;
 198
 199         /* wait_fd: create pipe */
 200         ret = pipe(waitfd);
 201         if (ret < 0) {
 202                 PERROR("pipe");
 203                 goto error_pipe;
 204         }
 205         for (i = 0; i < 2; i++) {
 206                 ret = fcntl(waitfd[i], F_SETFD, FD_CLOEXEC);
 207                 if (ret < 0) {
 208                         PERROR("fcntl");
 209                         goto error_fcntl;
 210                 }
 211         }
 212         /* The write end of the pipe needs to be non-blocking */
 213         ret = fcntl(waitfd[1], F_SETFL, O_NONBLOCK);
 214         if (ret < 0) {
 215                 PERROR("fcntl");
 216                 goto error_fcntl;
 217         }
 218         memcpy(obj->wait_fd, waitfd, sizeof(waitfd));
 219
 220         /* no shm_fd */
 221         obj->shm_fd = -1;
 222         obj->shm_fd_ownership = 0;
 223
 224         obj->type = SHM_OBJECT_MEM;
 225         obj->memory_map = memory_map;
 226         obj->memory_map_size = memory_map_size;
 227         obj->allocated_len = 0;
 228         obj->index = table->allocated_len++;
 229
 230         return obj;
 231
 232 error_fcntl:
 233         for (i = 0; i < 2; i++) {
 234                 ret = close(waitfd[i]);
 235                 if (ret) {
 236                         PERROR("close");
 237                         assert(0);
 238                 }
 239         }
 240 error_pipe:
 241         free(memory_map);
 242 alloc_error:
 243         return NULL;
 244 }
 245
 246 /*
 247  * libnuma prints errors on the console even for numa_available().
 248  * Work-around this limitation by using get_mempolicy() directly to
 249  * check whether the kernel supports mempolicy.
 250  */
 251 #ifdef HAVE_LIBNUMA
 252 static bool lttng_is_numa_available(void)
 253 {
 254         int ret;
 255
 256         ret = get_mempolicy(NULL, NULL, 0, NULL, 0);
 257         if (ret && errno == ENOSYS) {
 258                 return false;
 259         }
 260         return numa_available() > 0;
 261 }
 262 #endif
 263
 264 struct shm_object *shm_object_table_alloc(struct shm_object_table *table,
 265                         size_t memory_map_size,
 266                         enum shm_object_type type,
 267                         int stream_fd,
 268                         int cpu)
 269 {
 270         struct shm_object *shm_object;
 271 #ifdef HAVE_LIBNUMA
 272         int oldnode = 0, node;
 273         bool numa_avail;
 274
 275         numa_avail = lttng_is_numa_available();
 276         if (numa_avail) {
 277                 oldnode = numa_preferred();
 278                 if (cpu >= 0) {
 279                         node = numa_node_of_cpu(cpu);
 280                         if (node >= 0)
 281                                 numa_set_preferred(node);
 282                 }
 283                 if (cpu < 0 || node < 0)
 284                         numa_set_localalloc();
 285         }
 286 #endif /* HAVE_LIBNUMA */
 287         switch (type) {
 288         case SHM_OBJECT_SHM:
 289                 shm_object = _shm_object_table_alloc_shm(table, memory_map_size,
 290                                 stream_fd);
 291                 break;
 292         case SHM_OBJECT_MEM:
 293                 shm_object = _shm_object_table_alloc_mem(table, memory_map_size);
 294                 break;
 295         default:
 296                 assert(0);
 297         }
 298 #ifdef HAVE_LIBNUMA
 299         if (numa_avail)
 300                 numa_set_preferred(oldnode);
 301 #endif /* HAVE_LIBNUMA */
 302         return shm_object;
 303 }
 304
 305 struct shm_object *shm_object_table_append_shm(struct shm_object_table *table,
 306                         int shm_fd, int wakeup_fd, uint32_t stream_nr,
 307                         size_t memory_map_size)
 308 {
 309         struct shm_object *obj;
 310         char *memory_map;
 311         int ret;
 312
 313         if (table->allocated_len >= table->size)
 314                 return NULL;
 315         /* streams _must_ be received in sequential order, else fail. */
 316         if (stream_nr + 1 != table->allocated_len)
 317                 return NULL;
 318
 319         obj = &table->objects[table->allocated_len];
 320
 321         /* wait_fd: set write end of the pipe. */
 322         obj->wait_fd[0] = -1;   /* read end is unset */
 323         obj->wait_fd[1] = wakeup_fd;
 324         obj->shm_fd = shm_fd;
 325         obj->shm_fd_ownership = 1;
 326
 327         /* The write end of the pipe needs to be non-blocking */
 328         ret = fcntl(obj->wait_fd[1], F_SETFL, O_NONBLOCK);
 329         if (ret < 0) {
 330                 PERROR("fcntl");
 331                 goto error_fcntl;
 332         }
 333
 334         /* memory_map: mmap */
 335         memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
 336                           MAP_SHARED | LTTNG_MAP_POPULATE, shm_fd, 0);
 337         if (memory_map == MAP_FAILED) {
 338                 PERROR("mmap");
 339                 goto error_mmap;
 340         }
 341         obj->type = SHM_OBJECT_SHM;
 342         obj->memory_map = memory_map;
 343         obj->memory_map_size = memory_map_size;
 344         obj->allocated_len = memory_map_size;
 345         obj->index = table->allocated_len++;
 346
 347         return obj;
 348
 349 error_fcntl:
 350 error_mmap:
 351         return NULL;
 352 }
 353
 354 /*
 355  * Passing ownership of mem to object.
 356  */
 357 struct shm_object *shm_object_table_append_mem(struct shm_object_table *table,
 358                         void *mem, size_t memory_map_size, int wakeup_fd)
 359 {
 360         struct shm_object *obj;
 361         int ret;
 362
 363         if (table->allocated_len >= table->size)
 364                 return NULL;
 365         obj = &table->objects[table->allocated_len];
 366
 367         obj->wait_fd[0] = -1;   /* read end is unset */
 368         obj->wait_fd[1] = wakeup_fd;
 369         obj->shm_fd = -1;
 370         obj->shm_fd_ownership = 0;
 371
 372         ret = fcntl(obj->wait_fd[1], F_SETFD, FD_CLOEXEC);
 373         if (ret < 0) {
 374                 PERROR("fcntl");
 375                 goto error_fcntl;
 376         }
 377         /* The write end of the pipe needs to be non-blocking */
 378         ret = fcntl(obj->wait_fd[1], F_SETFL, O_NONBLOCK);
 379         if (ret < 0) {
 380                 PERROR("fcntl");
 381                 goto error_fcntl;
 382         }
 383
 384         obj->type = SHM_OBJECT_MEM;
 385         obj->memory_map = mem;
 386         obj->memory_map_size = memory_map_size;
 387         obj->allocated_len = memory_map_size;
 388         obj->index = table->allocated_len++;
 389
 390         return obj;
 391
 392 error_fcntl:
 393         return NULL;
 394 }
 395
 396 static
 397 void shmp_object_destroy(struct shm_object *obj, int consumer)
 398 {
 399         switch (obj->type) {
 400         case SHM_OBJECT_SHM:
 401         {
 402                 int ret, i;
 403
 404                 ret = munmap(obj->memory_map, obj->memory_map_size);
 405                 if (ret) {
 406                         PERROR("umnmap");
 407                         assert(0);
 408                 }
 409
 410                 if (obj->shm_fd_ownership) {
 411                         /* Delete FDs only if called from app (not consumer). */
 412                         if (!consumer) {
 413                                 lttng_ust_lock_fd_tracker();
 414                                 ret = close(obj->shm_fd);
 415                                 if (!ret) {
 416                                         lttng_ust_delete_fd_from_tracker(obj->shm_fd);
 417                                 } else {
 418                                         PERROR("close");
 419                                         assert(0);
 420                                 }
 421                                 lttng_ust_unlock_fd_tracker();
 422                         } else {
 423                                 ret = close(obj->shm_fd);
 424                                 if (ret) {
 425                                         PERROR("close");
 426                                         assert(0);
 427                                 }
 428                         }
 429                 }
 430                 for (i = 0; i < 2; i++) {
 431                         if (obj->wait_fd[i] < 0)
 432                                 continue;
 433                         if (!consumer) {
 434                                 lttng_ust_lock_fd_tracker();
 435                                 ret = close(obj->wait_fd[i]);
 436                                 if (!ret) {
 437                                         lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
 438                                 } else {
 439                                         PERROR("close");
 440                                         assert(0);
 441                                 }
 442                                 lttng_ust_unlock_fd_tracker();
 443                         } else {
 444                                 ret = close(obj->wait_fd[i]);
 445                                 if (ret) {
 446                                         PERROR("close");
 447                                         assert(0);
 448                                 }
 449                         }
 450                 }
 451                 break;
 452         }
 453         case SHM_OBJECT_MEM:
 454         {
 455                 int ret, i;
 456
 457                 for (i = 0; i < 2; i++) {
 458                         if (obj->wait_fd[i] < 0)
 459                                 continue;
 460                         if (!consumer) {
 461                                 lttng_ust_lock_fd_tracker();
 462                                 ret = close(obj->wait_fd[i]);
 463                                 if (!ret) {
 464                                         lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
 465                                 } else {
 466                                         PERROR("close");
 467                                         assert(0);
 468                                 }
 469                                 lttng_ust_unlock_fd_tracker();
 470                         } else {
 471                                 ret = close(obj->wait_fd[i]);
 472                                 if (ret) {
 473                                         PERROR("close");
 474                                         assert(0);
 475                                 }
 476                         }
 477                 }
 478                 free(obj->memory_map);
 479                 break;
 480         }
 481         default:
 482                 assert(0);
 483         }
 484 }
 485
 486 void shm_object_table_destroy(struct shm_object_table *table, int consumer)
 487 {
 488         int i;
 489
 490         for (i = 0; i < table->allocated_len; i++)
 491                 shmp_object_destroy(&table->objects[i], consumer);
 492         free(table);
 493 }
 494
 495 /*
 496  * zalloc_shm - allocate memory within a shm object.
 497  *
 498  * Shared memory is already zeroed by shmget.
 499  * *NOT* multithread-safe (should be protected by mutex).
 500  * Returns a -1, -1 tuple on error.
 501  */
 502 struct shm_ref zalloc_shm(struct shm_object *obj, size_t len)
 503 {
 504         struct shm_ref ref;
 505         struct shm_ref shm_ref_error = { -1, -1 };
 506
 507         if (obj->memory_map_size - obj->allocated_len < len)
 508                 return shm_ref_error;
 509         ref.index = obj->index;
 510         ref.offset =  obj->allocated_len;
 511         obj->allocated_len += len;
 512         return ref;
 513 }
 514
 515 void align_shm(struct shm_object *obj, size_t align)
 516 {
 517         size_t offset_len = lttng_ust_offset_align(obj->allocated_len, align);
 518         obj->allocated_len += offset_len;
 519 }