libringbuffer/shm.c

   1 /*
   2  * libringbuffer/shm.c
   3  *
   4  * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; only
   9  * version 2.1 of the License.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #define _LGPL_SOURCE
  22 #include <config.h>
  23 #include "shm.h"
  24 #include <unistd.h>
  25 #include <fcntl.h>
  26 #include <sys/mman.h>
  27 #include <sys/types.h>
  28 #include <sys/stat.h>   /* For mode constants */
  29 #include <fcntl.h>      /* For O_* constants */
  30 #include <assert.h>
  31 #include <stdio.h>
  32 #include <signal.h>
  33 #include <dirent.h>
  34 #include <lttng/align.h>
  35 #include <limits.h>
  36 #include <stdbool.h>
  37 #ifdef HAVE_LIBNUMA
  38 #include <numa.h>
  39 #include <numaif.h>
  40 #endif
  41 #include <helper.h>
  42 #include <ust-fd.h>
  43
  44 /*
  45  * Ensure we have the required amount of space available by writing 0
  46  * into the entire buffer. Not doing so can trigger SIGBUS when going
  47  * beyond the available shm space.
  48  */
  49 static
  50 int zero_file(int fd, size_t len)
  51 {
  52         ssize_t retlen;
  53         size_t written = 0;
  54         char *zeropage;
  55         long pagelen;
  56         int ret;
  57
  58         pagelen = sysconf(_SC_PAGESIZE);
  59         if (pagelen < 0)
  60                 return (int) pagelen;
  61         zeropage = calloc(pagelen, 1);
  62         if (!zeropage)
  63                 return -ENOMEM;
  64
  65         while (len > written) {
  66                 do {
  67                         retlen = write(fd, zeropage,
  68                                 min_t(size_t, pagelen, len - written));
  69                 } while (retlen == -1UL && errno == EINTR);
  70                 if (retlen < 0) {
  71                         ret = (int) retlen;
  72                         goto error;
  73                 }
  74                 written += retlen;
  75         }
  76         ret = 0;
  77 error:
  78         free(zeropage);
  79         return ret;
  80 }
  81
  82 struct shm_object_table *shm_object_table_create(size_t max_nb_obj)
  83 {
  84         struct shm_object_table *table;
  85
  86         table = zmalloc(sizeof(struct shm_object_table) +
  87                         max_nb_obj * sizeof(table->objects[0]));
  88         if (!table)
  89                 return NULL;
  90         table->size = max_nb_obj;
  91         return table;
  92 }
  93
  94 static
  95 struct shm_object *_shm_object_table_alloc_shm(struct shm_object_table *table,
  96                                            size_t memory_map_size,
  97                                            int stream_fd)
  98 {
  99         int shmfd, waitfd[2], ret, i;
 100         struct shm_object *obj;
 101         char *memory_map;
 102
 103         if (stream_fd < 0)
 104                 return NULL;
 105         if (table->allocated_len >= table->size)
 106                 return NULL;
 107         obj = &table->objects[table->allocated_len];
 108
 109         /* wait_fd: create pipe */
 110         ret = pipe(waitfd);
 111         if (ret < 0) {
 112                 PERROR("pipe");
 113                 goto error_pipe;
 114         }
 115         for (i = 0; i < 2; i++) {
 116                 ret = fcntl(waitfd[i], F_SETFD, FD_CLOEXEC);
 117                 if (ret < 0) {
 118                         PERROR("fcntl");
 119                         goto error_fcntl;
 120                 }
 121         }
 122         /* The write end of the pipe needs to be non-blocking */
 123         ret = fcntl(waitfd[1], F_SETFL, O_NONBLOCK);
 124         if (ret < 0) {
 125                 PERROR("fcntl");
 126                 goto error_fcntl;
 127         }
 128         memcpy(obj->wait_fd, waitfd, sizeof(waitfd));
 129
 130         /* create shm */
 131
 132         shmfd = stream_fd;
 133         ret = zero_file(shmfd, memory_map_size);
 134         if (ret) {
 135                 PERROR("zero_file");
 136                 goto error_zero_file;
 137         }
 138         ret = ftruncate(shmfd, memory_map_size);
 139         if (ret) {
 140                 PERROR("ftruncate");
 141                 goto error_ftruncate;
 142         }
 143         /*
 144          * Also ensure the file metadata is synced with the storage by using
 145          * fsync(2).
 146          */
 147         ret = fsync(shmfd);
 148         if (ret) {
 149                 PERROR("fsync");
 150                 goto error_fsync;
 151         }
 152         obj->shm_fd_ownership = 0;
 153         obj->shm_fd = shmfd;
 154
 155         /* memory_map: mmap */
 156         memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
 157                           MAP_SHARED, shmfd, 0);
 158         if (memory_map == MAP_FAILED) {
 159                 PERROR("mmap");
 160                 goto error_mmap;
 161         }
 162         obj->type = SHM_OBJECT_SHM;
 163         obj->memory_map = memory_map;
 164         obj->memory_map_size = memory_map_size;
 165         obj->allocated_len = 0;
 166         obj->index = table->allocated_len++;
 167
 168         return obj;
 169
 170 error_mmap:
 171 error_fsync:
 172 error_ftruncate:
 173 error_zero_file:
 174 error_fcntl:
 175         for (i = 0; i < 2; i++) {
 176                 ret = close(waitfd[i]);
 177                 if (ret) {
 178                         PERROR("close");
 179                         assert(0);
 180                 }
 181         }
 182 error_pipe:
 183         return NULL;
 184 }
 185
 186 static
 187 struct shm_object *_shm_object_table_alloc_mem(struct shm_object_table *table,
 188                                            size_t memory_map_size)
 189 {
 190         struct shm_object *obj;
 191         void *memory_map;
 192         int waitfd[2], i, ret;
 193
 194         if (table->allocated_len >= table->size)
 195                 return NULL;
 196         obj = &table->objects[table->allocated_len];
 197
 198         memory_map = zmalloc(memory_map_size);
 199         if (!memory_map)
 200                 goto alloc_error;
 201
 202         /* wait_fd: create pipe */
 203         ret = pipe(waitfd);
 204         if (ret < 0) {
 205                 PERROR("pipe");
 206                 goto error_pipe;
 207         }
 208         for (i = 0; i < 2; i++) {
 209                 ret = fcntl(waitfd[i], F_SETFD, FD_CLOEXEC);
 210                 if (ret < 0) {
 211                         PERROR("fcntl");
 212                         goto error_fcntl;
 213                 }
 214         }
 215         /* The write end of the pipe needs to be non-blocking */
 216         ret = fcntl(waitfd[1], F_SETFL, O_NONBLOCK);
 217         if (ret < 0) {
 218                 PERROR("fcntl");
 219                 goto error_fcntl;
 220         }
 221         memcpy(obj->wait_fd, waitfd, sizeof(waitfd));
 222
 223         /* no shm_fd */
 224         obj->shm_fd = -1;
 225         obj->shm_fd_ownership = 0;
 226
 227         obj->type = SHM_OBJECT_MEM;
 228         obj->memory_map = memory_map;
 229         obj->memory_map_size = memory_map_size;
 230         obj->allocated_len = 0;
 231         obj->index = table->allocated_len++;
 232
 233         return obj;
 234
 235 error_fcntl:
 236         for (i = 0; i < 2; i++) {
 237                 ret = close(waitfd[i]);
 238                 if (ret) {
 239                         PERROR("close");
 240                         assert(0);
 241                 }
 242         }
 243 error_pipe:
 244         free(memory_map);
 245 alloc_error:
 246         return NULL;
 247 }
 248
 249 /*
 250  * libnuma prints errors on the console even for numa_available().
 251  * Work-around this limitation by using get_mempolicy() directly to
 252  * check whether the kernel supports mempolicy.
 253  */
 254 #ifdef HAVE_LIBNUMA
 255 static bool lttng_is_numa_available(void)
 256 {
 257         int ret;
 258
 259         ret = get_mempolicy(NULL, NULL, 0, NULL, 0);
 260         if (ret && errno == ENOSYS) {
 261                 return false;
 262         }
 263         return numa_available() > 0;
 264 }
 265 #endif
 266
 267 struct shm_object *shm_object_table_alloc(struct shm_object_table *table,
 268                         size_t memory_map_size,
 269                         enum shm_object_type type,
 270                         int stream_fd,
 271                         int cpu)
 272 {
 273         struct shm_object *shm_object;
 274 #ifdef HAVE_LIBNUMA
 275         int oldnode = 0, node;
 276         bool numa_avail;
 277
 278         numa_avail = lttng_is_numa_available();
 279         if (numa_avail) {
 280                 oldnode = numa_preferred();
 281                 if (cpu >= 0) {
 282                         node = numa_node_of_cpu(cpu);
 283                         if (node >= 0)
 284                                 numa_set_preferred(node);
 285                 }
 286                 if (cpu < 0 || node < 0)
 287                         numa_set_localalloc();
 288         }
 289 #endif /* HAVE_LIBNUMA */
 290         switch (type) {
 291         case SHM_OBJECT_SHM:
 292                 shm_object = _shm_object_table_alloc_shm(table, memory_map_size,
 293                                 stream_fd);
 294                 break;
 295         case SHM_OBJECT_MEM:
 296                 shm_object = _shm_object_table_alloc_mem(table, memory_map_size);
 297                 break;
 298         default:
 299                 assert(0);
 300         }
 301 #ifdef HAVE_LIBNUMA
 302         if (numa_avail)
 303                 numa_set_preferred(oldnode);
 304 #endif /* HAVE_LIBNUMA */
 305         return shm_object;
 306 }
 307
 308 struct shm_object *shm_object_table_append_shm(struct shm_object_table *table,
 309                         int shm_fd, int wakeup_fd, uint32_t stream_nr,
 310                         size_t memory_map_size)
 311 {
 312         struct shm_object *obj;
 313         char *memory_map;
 314         int ret;
 315
 316         if (table->allocated_len >= table->size)
 317                 return NULL;
 318         /* streams _must_ be received in sequential order, else fail. */
 319         if (stream_nr + 1 != table->allocated_len)
 320                 return NULL;
 321
 322         obj = &table->objects[table->allocated_len];
 323
 324         /* wait_fd: set write end of the pipe. */
 325         obj->wait_fd[0] = -1;   /* read end is unset */
 326         obj->wait_fd[1] = wakeup_fd;
 327         obj->shm_fd = shm_fd;
 328         obj->shm_fd_ownership = 1;
 329
 330         ret = fcntl(obj->wait_fd[1], F_SETFD, FD_CLOEXEC);
 331         if (ret < 0) {
 332                 PERROR("fcntl");
 333                 goto error_fcntl;
 334         }
 335         /* The write end of the pipe needs to be non-blocking */
 336         ret = fcntl(obj->wait_fd[1], F_SETFL, O_NONBLOCK);
 337         if (ret < 0) {
 338                 PERROR("fcntl");
 339                 goto error_fcntl;
 340         }
 341
 342         /* memory_map: mmap */
 343         memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
 344                           MAP_SHARED, shm_fd, 0);
 345         if (memory_map == MAP_FAILED) {
 346                 PERROR("mmap");
 347                 goto error_mmap;
 348         }
 349         obj->type = SHM_OBJECT_SHM;
 350         obj->memory_map = memory_map;
 351         obj->memory_map_size = memory_map_size;
 352         obj->allocated_len = memory_map_size;
 353         obj->index = table->allocated_len++;
 354
 355         return obj;
 356
 357 error_fcntl:
 358 error_mmap:
 359         return NULL;
 360 }
 361
 362 /*
 363  * Passing ownership of mem to object.
 364  */
 365 struct shm_object *shm_object_table_append_mem(struct shm_object_table *table,
 366                         void *mem, size_t memory_map_size, int wakeup_fd)
 367 {
 368         struct shm_object *obj;
 369         int ret;
 370
 371         if (table->allocated_len >= table->size)
 372                 return NULL;
 373         obj = &table->objects[table->allocated_len];
 374
 375         obj->wait_fd[0] = -1;   /* read end is unset */
 376         obj->wait_fd[1] = wakeup_fd;
 377         obj->shm_fd = -1;
 378         obj->shm_fd_ownership = 0;
 379
 380         ret = fcntl(obj->wait_fd[1], F_SETFD, FD_CLOEXEC);
 381         if (ret < 0) {
 382                 PERROR("fcntl");
 383                 goto error_fcntl;
 384         }
 385         /* The write end of the pipe needs to be non-blocking */
 386         ret = fcntl(obj->wait_fd[1], F_SETFL, O_NONBLOCK);
 387         if (ret < 0) {
 388                 PERROR("fcntl");
 389                 goto error_fcntl;
 390         }
 391
 392         obj->type = SHM_OBJECT_MEM;
 393         obj->memory_map = mem;
 394         obj->memory_map_size = memory_map_size;
 395         obj->allocated_len = memory_map_size;
 396         obj->index = table->allocated_len++;
 397
 398         return obj;
 399
 400 error_fcntl:
 401         return NULL;
 402 }
 403
 404 static
 405 void shmp_object_destroy(struct shm_object *obj, int consumer)
 406 {
 407         switch (obj->type) {
 408         case SHM_OBJECT_SHM:
 409         {
 410                 int ret, i;
 411
 412                 ret = munmap(obj->memory_map, obj->memory_map_size);
 413                 if (ret) {
 414                         PERROR("umnmap");
 415                         assert(0);
 416                 }
 417
 418                 if (obj->shm_fd_ownership) {
 419                         /* Delete FDs only if called from app (not consumer). */
 420                         if (!consumer) {
 421                                 lttng_ust_lock_fd_tracker();
 422                                 ret = close(obj->shm_fd);
 423                                 if (!ret) {
 424                                         lttng_ust_delete_fd_from_tracker(obj->shm_fd);
 425                                 } else {
 426                                         PERROR("close");
 427                                         assert(0);
 428                                 }
 429                                 lttng_ust_unlock_fd_tracker();
 430                         } else {
 431                                 ret = close(obj->shm_fd);
 432                                 if (ret) {
 433                                         PERROR("close");
 434                                         assert(0);
 435                                 }
 436                         }
 437                 }
 438                 for (i = 0; i < 2; i++) {
 439                         if (obj->wait_fd[i] < 0)
 440                                 continue;
 441                         if (!consumer) {
 442                                 lttng_ust_lock_fd_tracker();
 443                                 ret = close(obj->wait_fd[i]);
 444                                 if (!ret) {
 445                                         lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
 446                                 } else {
 447                                         PERROR("close");
 448                                         assert(0);
 449                                 }
 450                                 lttng_ust_unlock_fd_tracker();
 451                         } else {
 452                                 ret = close(obj->wait_fd[i]);
 453                                 if (ret) {
 454                                         PERROR("close");
 455                                         assert(0);
 456                                 }
 457                         }
 458                 }
 459                 break;
 460         }
 461         case SHM_OBJECT_MEM:
 462         {
 463                 int ret, i;
 464
 465                 for (i = 0; i < 2; i++) {
 466                         if (obj->wait_fd[i] < 0)
 467                                 continue;
 468                         if (!consumer) {
 469                                 lttng_ust_lock_fd_tracker();
 470                                 ret = close(obj->wait_fd[i]);
 471                                 if (!ret) {
 472                                         lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
 473                                 } else {
 474                                         PERROR("close");
 475                                         assert(0);
 476                                 }
 477                                 lttng_ust_unlock_fd_tracker();
 478                         } else {
 479                                 ret = close(obj->wait_fd[i]);
 480                                 if (ret) {
 481                                         PERROR("close");
 482                                         assert(0);
 483                                 }
 484                         }
 485                 }
 486                 free(obj->memory_map);
 487                 break;
 488         }
 489         default:
 490                 assert(0);
 491         }
 492 }
 493
 494 void shm_object_table_destroy(struct shm_object_table *table, int consumer)
 495 {
 496         int i;
 497
 498         for (i = 0; i < table->allocated_len; i++)
 499                 shmp_object_destroy(&table->objects[i], consumer);
 500         free(table);
 501 }
 502
 503 /*
 504  * zalloc_shm - allocate memory within a shm object.
 505  *
 506  * Shared memory is already zeroed by shmget.
 507  * *NOT* multithread-safe (should be protected by mutex).
 508  * Returns a -1, -1 tuple on error.
 509  */
 510 struct shm_ref zalloc_shm(struct shm_object *obj, size_t len)
 511 {
 512         struct shm_ref ref;
 513         struct shm_ref shm_ref_error = { -1, -1 };
 514
 515         if (obj->memory_map_size - obj->allocated_len < len)
 516                 return shm_ref_error;
 517         ref.index = obj->index;
 518         ref.offset =  obj->allocated_len;
 519         obj->allocated_len += len;
 520         return ref;
 521 }
 522
 523 void align_shm(struct shm_object *obj, size_t align)
 524 {
 525         size_t offset_len = offset_align(obj->allocated_len, align);
 526         obj->allocated_len += offset_len;
 527 }