libringbuffer/shm.c

   1 /*
   2  * libringbuffer/shm.c
   3  *
   4  * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; only
   9  * version 2.1 of the License.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #define _LGPL_SOURCE
  22 #include "shm.h"
  23 #include <unistd.h>
  24 #include <fcntl.h>
  25 #include <sys/mman.h>
  26 #include <sys/types.h>
  27 #include <sys/stat.h>   /* For mode constants */
  28 #include <fcntl.h>      /* For O_* constants */
  29 #include <assert.h>
  30 #include <stdio.h>
  31 #include <signal.h>
  32 #include <dirent.h>
  33 #include <lttng/align.h>
  34 #include <limits.h>
  35 #include <numa.h>
  36 #include <helper.h>
  37 #include <ust-fd.h>
  38
  39 /*
  40  * Ensure we have the required amount of space available by writing 0
  41  * into the entire buffer. Not doing so can trigger SIGBUS when going
  42  * beyond the available shm space.
  43  */
  44 static
  45 int zero_file(int fd, size_t len)
  46 {
  47         ssize_t retlen;
  48         size_t written = 0;
  49         char *zeropage;
  50         long pagelen;
  51         int ret;
  52
  53         pagelen = sysconf(_SC_PAGESIZE);
  54         if (pagelen < 0)
  55                 return (int) pagelen;
  56         zeropage = calloc(pagelen, 1);
  57         if (!zeropage)
  58                 return -ENOMEM;
  59
  60         while (len > written) {
  61                 do {
  62                         retlen = write(fd, zeropage,
  63                                 min_t(size_t, pagelen, len - written));
  64                 } while (retlen == -1UL && errno == EINTR);
  65                 if (retlen < 0) {
  66                         ret = (int) retlen;
  67                         goto error;
  68                 }
  69                 written += retlen;
  70         }
  71         ret = 0;
  72 error:
  73         free(zeropage);
  74         return ret;
  75 }
  76
  77 struct shm_object_table *shm_object_table_create(size_t max_nb_obj)
  78 {
  79         struct shm_object_table *table;
  80
  81         table = zmalloc(sizeof(struct shm_object_table) +
  82                         max_nb_obj * sizeof(table->objects[0]));
  83         if (!table)
  84                 return NULL;
  85         table->size = max_nb_obj;
  86         return table;
  87 }
  88
  89 static
  90 struct shm_object *_shm_object_table_alloc_shm(struct shm_object_table *table,
  91                                            size_t memory_map_size,
  92                                            int stream_fd)
  93 {
  94         int shmfd, waitfd[2], ret, i;
  95         struct shm_object *obj;
  96         char *memory_map;
  97
  98         if (stream_fd < 0)
  99                 return NULL;
 100         if (table->allocated_len >= table->size)
 101                 return NULL;
 102         obj = &table->objects[table->allocated_len];
 103
 104         /* wait_fd: create pipe */
 105         ret = pipe(waitfd);
 106         if (ret < 0) {
 107                 PERROR("pipe");
 108                 goto error_pipe;
 109         }
 110         for (i = 0; i < 2; i++) {
 111                 ret = fcntl(waitfd[i], F_SETFD, FD_CLOEXEC);
 112                 if (ret < 0) {
 113                         PERROR("fcntl");
 114                         goto error_fcntl;
 115                 }
 116         }
 117         /* The write end of the pipe needs to be non-blocking */
 118         ret = fcntl(waitfd[1], F_SETFL, O_NONBLOCK);
 119         if (ret < 0) {
 120                 PERROR("fcntl");
 121                 goto error_fcntl;
 122         }
 123         memcpy(obj->wait_fd, waitfd, sizeof(waitfd));
 124
 125         /* create shm */
 126
 127         shmfd = stream_fd;
 128         ret = zero_file(shmfd, memory_map_size);
 129         if (ret) {
 130                 PERROR("zero_file");
 131                 goto error_zero_file;
 132         }
 133         ret = ftruncate(shmfd, memory_map_size);
 134         if (ret) {
 135                 PERROR("ftruncate");
 136                 goto error_ftruncate;
 137         }
 138         /*
 139          * Also ensure the file metadata is synced with the storage by using
 140          * fsync(2).
 141          */
 142         ret = fsync(shmfd);
 143         if (ret) {
 144                 PERROR("fsync");
 145                 goto error_fsync;
 146         }
 147         obj->shm_fd_ownership = 0;
 148         obj->shm_fd = shmfd;
 149
 150         /* memory_map: mmap */
 151         memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
 152                           MAP_SHARED, shmfd, 0);
 153         if (memory_map == MAP_FAILED) {
 154                 PERROR("mmap");
 155                 goto error_mmap;
 156         }
 157         obj->type = SHM_OBJECT_SHM;
 158         obj->memory_map = memory_map;
 159         obj->memory_map_size = memory_map_size;
 160         obj->allocated_len = 0;
 161         obj->index = table->allocated_len++;
 162
 163         return obj;
 164
 165 error_mmap:
 166 error_fsync:
 167 error_ftruncate:
 168 error_zero_file:
 169 error_fcntl:
 170         for (i = 0; i < 2; i++) {
 171                 ret = close(waitfd[i]);
 172                 if (ret) {
 173                         PERROR("close");
 174                         assert(0);
 175                 }
 176         }
 177 error_pipe:
 178         return NULL;
 179 }
 180
 181 static
 182 struct shm_object *_shm_object_table_alloc_mem(struct shm_object_table *table,
 183                                            size_t memory_map_size)
 184 {
 185         struct shm_object *obj;
 186         void *memory_map;
 187         int waitfd[2], i, ret;
 188
 189         if (table->allocated_len >= table->size)
 190                 return NULL;
 191         obj = &table->objects[table->allocated_len];
 192
 193         memory_map = zmalloc(memory_map_size);
 194         if (!memory_map)
 195                 goto alloc_error;
 196
 197         /* wait_fd: create pipe */
 198         ret = pipe(waitfd);
 199         if (ret < 0) {
 200                 PERROR("pipe");
 201                 goto error_pipe;
 202         }
 203         for (i = 0; i < 2; i++) {
 204                 ret = fcntl(waitfd[i], F_SETFD, FD_CLOEXEC);
 205                 if (ret < 0) {
 206                         PERROR("fcntl");
 207                         goto error_fcntl;
 208                 }
 209         }
 210         /* The write end of the pipe needs to be non-blocking */
 211         ret = fcntl(waitfd[1], F_SETFL, O_NONBLOCK);
 212         if (ret < 0) {
 213                 PERROR("fcntl");
 214                 goto error_fcntl;
 215         }
 216         memcpy(obj->wait_fd, waitfd, sizeof(waitfd));
 217
 218         /* no shm_fd */
 219         obj->shm_fd = -1;
 220         obj->shm_fd_ownership = 0;
 221
 222         obj->type = SHM_OBJECT_MEM;
 223         obj->memory_map = memory_map;
 224         obj->memory_map_size = memory_map_size;
 225         obj->allocated_len = 0;
 226         obj->index = table->allocated_len++;
 227
 228         return obj;
 229
 230 error_fcntl:
 231         for (i = 0; i < 2; i++) {
 232                 ret = close(waitfd[i]);
 233                 if (ret) {
 234                         PERROR("close");
 235                         assert(0);
 236                 }
 237         }
 238 error_pipe:
 239         free(memory_map);
 240 alloc_error:
 241         return NULL;
 242 }
 243
 244 struct shm_object *shm_object_table_alloc(struct shm_object_table *table,
 245                         size_t memory_map_size,
 246                         enum shm_object_type type,
 247                         int stream_fd,
 248                         int cpu)
 249 {
 250         int oldnode, node;
 251         struct shm_object *shm_object;
 252
 253         oldnode = numa_preferred();
 254         if (cpu >= 0) {
 255                 node = numa_node_of_cpu(cpu);
 256                 if (node >= 0)
 257                         numa_set_preferred(node);
 258         }
 259         if (cpu < 0 || node < 0)
 260                 numa_set_localalloc();
 261         switch (type) {
 262         case SHM_OBJECT_SHM:
 263                 shm_object = _shm_object_table_alloc_shm(table, memory_map_size,
 264                                 stream_fd);
 265                 break;
 266         case SHM_OBJECT_MEM:
 267                 shm_object = _shm_object_table_alloc_mem(table, memory_map_size);
 268                 break;
 269         default:
 270                 assert(0);
 271         }
 272         numa_set_preferred(oldnode);
 273         return shm_object;
 274 }
 275
 276 struct shm_object *shm_object_table_append_shm(struct shm_object_table *table,
 277                         int shm_fd, int wakeup_fd, uint32_t stream_nr,
 278                         size_t memory_map_size)
 279 {
 280         struct shm_object *obj;
 281         char *memory_map;
 282         int ret;
 283
 284         if (table->allocated_len >= table->size)
 285                 return NULL;
 286         /* streams _must_ be received in sequential order, else fail. */
 287         if (stream_nr + 1 != table->allocated_len)
 288                 return NULL;
 289
 290         obj = &table->objects[table->allocated_len];
 291
 292         /* wait_fd: set write end of the pipe. */
 293         obj->wait_fd[0] = -1;   /* read end is unset */
 294         obj->wait_fd[1] = wakeup_fd;
 295         obj->shm_fd = shm_fd;
 296         obj->shm_fd_ownership = 1;
 297
 298         ret = fcntl(obj->wait_fd[1], F_SETFD, FD_CLOEXEC);
 299         if (ret < 0) {
 300                 PERROR("fcntl");
 301                 goto error_fcntl;
 302         }
 303         /* The write end of the pipe needs to be non-blocking */
 304         ret = fcntl(obj->wait_fd[1], F_SETFL, O_NONBLOCK);
 305         if (ret < 0) {
 306                 PERROR("fcntl");
 307                 goto error_fcntl;
 308         }
 309
 310         /* memory_map: mmap */
 311         memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
 312                           MAP_SHARED, shm_fd, 0);
 313         if (memory_map == MAP_FAILED) {
 314                 PERROR("mmap");
 315                 goto error_mmap;
 316         }
 317         obj->type = SHM_OBJECT_SHM;
 318         obj->memory_map = memory_map;
 319         obj->memory_map_size = memory_map_size;
 320         obj->allocated_len = memory_map_size;
 321         obj->index = table->allocated_len++;
 322
 323         return obj;
 324
 325 error_fcntl:
 326 error_mmap:
 327         return NULL;
 328 }
 329
 330 /*
 331  * Passing ownership of mem to object.
 332  */
 333 struct shm_object *shm_object_table_append_mem(struct shm_object_table *table,
 334                         void *mem, size_t memory_map_size, int wakeup_fd)
 335 {
 336         struct shm_object *obj;
 337         int ret;
 338
 339         if (table->allocated_len >= table->size)
 340                 return NULL;
 341         obj = &table->objects[table->allocated_len];
 342
 343         obj->wait_fd[0] = -1;   /* read end is unset */
 344         obj->wait_fd[1] = wakeup_fd;
 345         obj->shm_fd = -1;
 346         obj->shm_fd_ownership = 0;
 347
 348         ret = fcntl(obj->wait_fd[1], F_SETFD, FD_CLOEXEC);
 349         if (ret < 0) {
 350                 PERROR("fcntl");
 351                 goto error_fcntl;
 352         }
 353         /* The write end of the pipe needs to be non-blocking */
 354         ret = fcntl(obj->wait_fd[1], F_SETFL, O_NONBLOCK);
 355         if (ret < 0) {
 356                 PERROR("fcntl");
 357                 goto error_fcntl;
 358         }
 359
 360         obj->type = SHM_OBJECT_MEM;
 361         obj->memory_map = mem;
 362         obj->memory_map_size = memory_map_size;
 363         obj->allocated_len = memory_map_size;
 364         obj->index = table->allocated_len++;
 365
 366         return obj;
 367
 368 error_fcntl:
 369         return NULL;
 370 }
 371
 372 static
 373 void shmp_object_destroy(struct shm_object *obj, int consumer)
 374 {
 375         switch (obj->type) {
 376         case SHM_OBJECT_SHM:
 377         {
 378                 int ret, i;
 379
 380                 ret = munmap(obj->memory_map, obj->memory_map_size);
 381                 if (ret) {
 382                         PERROR("umnmap");
 383                         assert(0);
 384                 }
 385
 386                 if (obj->shm_fd_ownership) {
 387                         /* Delete FDs only if called from app (not consumer). */
 388                         if (!consumer) {
 389                                 lttng_ust_lock_fd_tracker();
 390                                 ret = close(obj->shm_fd);
 391                                 if (!ret) {
 392                                         lttng_ust_delete_fd_from_tracker(obj->shm_fd);
 393                                 } else {
 394                                         PERROR("close");
 395                                         assert(0);
 396                                 }
 397                                 lttng_ust_unlock_fd_tracker();
 398                         } else {
 399                                 ret = close(obj->shm_fd);
 400                                 if (ret) {
 401                                         PERROR("close");
 402                                         assert(0);
 403                                 }
 404                         }
 405                 }
 406                 for (i = 0; i < 2; i++) {
 407                         if (obj->wait_fd[i] < 0)
 408                                 continue;
 409                         if (!consumer) {
 410                                 lttng_ust_lock_fd_tracker();
 411                                 ret = close(obj->wait_fd[i]);
 412                                 if (!ret) {
 413                                         lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
 414                                 } else {
 415                                         PERROR("close");
 416                                         assert(0);
 417                                 }
 418                                 lttng_ust_unlock_fd_tracker();
 419                         } else {
 420                                 ret = close(obj->wait_fd[i]);
 421                                 if (ret) {
 422                                         PERROR("close");
 423                                         assert(0);
 424                                 }
 425                         }
 426                 }
 427                 break;
 428         }
 429         case SHM_OBJECT_MEM:
 430         {
 431                 int ret, i;
 432
 433                 for (i = 0; i < 2; i++) {
 434                         if (obj->wait_fd[i] < 0)
 435                                 continue;
 436                         if (!consumer) {
 437                                 lttng_ust_lock_fd_tracker();
 438                                 ret = close(obj->wait_fd[i]);
 439                                 if (!ret) {
 440                                         lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
 441                                 } else {
 442                                         PERROR("close");
 443                                         assert(0);
 444                                 }
 445                                 lttng_ust_unlock_fd_tracker();
 446                         } else {
 447                                 ret = close(obj->wait_fd[i]);
 448                                 if (ret) {
 449                                         PERROR("close");
 450                                         assert(0);
 451                                 }
 452                         }
 453                 }
 454                 free(obj->memory_map);
 455                 break;
 456         }
 457         default:
 458                 assert(0);
 459         }
 460 }
 461
 462 void shm_object_table_destroy(struct shm_object_table *table, int consumer)
 463 {
 464         int i;
 465
 466         for (i = 0; i < table->allocated_len; i++)
 467                 shmp_object_destroy(&table->objects[i], consumer);
 468         free(table);
 469 }
 470
 471 /*
 472  * zalloc_shm - allocate memory within a shm object.
 473  *
 474  * Shared memory is already zeroed by shmget.
 475  * *NOT* multithread-safe (should be protected by mutex).
 476  * Returns a -1, -1 tuple on error.
 477  */
 478 struct shm_ref zalloc_shm(struct shm_object *obj, size_t len)
 479 {
 480         struct shm_ref ref;
 481         struct shm_ref shm_ref_error = { -1, -1 };
 482
 483         if (obj->memory_map_size - obj->allocated_len < len)
 484                 return shm_ref_error;
 485         ref.index = obj->index;
 486         ref.offset =  obj->allocated_len;
 487         obj->allocated_len += len;
 488         return ref;
 489 }
 490
 491 void align_shm(struct shm_object *obj, size_t align)
 492 {
 493         size_t offset_len = offset_align(obj->allocated_len, align);
 494         obj->allocated_len += offset_len;
 495 }