Use MAP_POPULATE to reduce pagefault when available

[lttng-ust.git] / libringbuffer / shm.c

diff --git a/libringbuffer/shm.c b/libringbuffer/shm.c
index 86e84b3f34a76bf28772602d50b99cb2c3d99f56..909991ed27c6f1ada4a04426e396d232d126df8e 100644 (file)
--- a/libringbuffer/shm.c
+++ b/libringbuffer/shm.c
@@ -19,6 +19,7 @@
  */
 
 #define _LGPL_SOURCE
+#include <config.h>
 #include "shm.h"
 #include <unistd.h>
 #include <fcntl.h>
@@ -32,7 +33,14 @@
 #include <dirent.h>
 #include <lttng/align.h>
 #include <limits.h>
+#include <stdbool.h>
+#ifdef HAVE_LIBNUMA
+#include <numa.h>
+#include <numaif.h>
+#endif
 #include <helper.h>
+#include <ust-fd.h>
+#include "mmap.h"
 
 /*
  * Ensure we have the required amount of space available by writing 0
@@ -133,12 +141,21 @@ struct shm_object *_shm_object_table_alloc_shm(struct shm_object_table *table,
                PERROR("ftruncate");
                goto error_ftruncate;
        }
+       /*
+        * Also ensure the file metadata is synced with the storage by using
+        * fsync(2).
+        */
+       ret = fsync(shmfd);
+       if (ret) {
+               PERROR("fsync");
+               goto error_fsync;
+       }
        obj->shm_fd_ownership = 0;
        obj->shm_fd = shmfd;
 
        /* memory_map: mmap */
        memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
-                         MAP_SHARED, shmfd, 0);
+                         MAP_SHARED | LTTNG_MAP_POPULATE, shmfd, 0);
        if (memory_map == MAP_FAILED) {
                PERROR("mmap");
                goto error_mmap;
@@ -152,6 +169,7 @@ struct shm_object *_shm_object_table_alloc_shm(struct shm_object_table *table,
        return obj;
 
 error_mmap:
+error_fsync:
 error_ftruncate:
 error_zero_file:
 error_fcntl:
@@ -229,21 +247,63 @@ alloc_error:
        return NULL;
 }
 
+/*
+ * libnuma prints errors on the console even for numa_available().
+ * Work-around this limitation by using get_mempolicy() directly to
+ * check whether the kernel supports mempolicy.
+ */
+#ifdef HAVE_LIBNUMA
+static bool lttng_is_numa_available(void)
+{
+       int ret;
+
+       ret = get_mempolicy(NULL, NULL, 0, NULL, 0);
+       if (ret && errno == ENOSYS) {
+               return false;
+       }
+       return numa_available() > 0;
+}
+#endif
+
 struct shm_object *shm_object_table_alloc(struct shm_object_table *table,
                        size_t memory_map_size,
                        enum shm_object_type type,
-                       int stream_fd)
+                       int stream_fd,
+                       int cpu)
 {
+       struct shm_object *shm_object;
+#ifdef HAVE_LIBNUMA
+       int oldnode = 0, node;
+       bool numa_avail;
+
+       numa_avail = lttng_is_numa_available();
+       if (numa_avail) {
+               oldnode = numa_preferred();
+               if (cpu >= 0) {
+                       node = numa_node_of_cpu(cpu);
+                       if (node >= 0)
+                               numa_set_preferred(node);
+               }
+               if (cpu < 0 || node < 0)
+                       numa_set_localalloc();
+       }
+#endif /* HAVE_LIBNUMA */
        switch (type) {
        case SHM_OBJECT_SHM:
-               return _shm_object_table_alloc_shm(table, memory_map_size,
+               shm_object = _shm_object_table_alloc_shm(table, memory_map_size,
                                stream_fd);
+               break;
        case SHM_OBJECT_MEM:
-               return _shm_object_table_alloc_mem(table, memory_map_size);
+               shm_object = _shm_object_table_alloc_mem(table, memory_map_size);
+               break;
        default:
                assert(0);
        }
-       return NULL;
+#ifdef HAVE_LIBNUMA
+       if (numa_avail)
+               numa_set_preferred(oldnode);
+#endif /* HAVE_LIBNUMA */
+       return shm_object;
 }
 
 struct shm_object *shm_object_table_append_shm(struct shm_object_table *table,
@@ -282,7 +342,7 @@ struct shm_object *shm_object_table_append_shm(struct shm_object_table *table,
 
        /* memory_map: mmap */
        memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
-                         MAP_SHARED, shm_fd, 0);
+                         MAP_SHARED | LTTNG_MAP_POPULATE, shm_fd, 0);
        if (memory_map == MAP_FAILED) {
                PERROR("mmap");
                goto error_mmap;
@@ -343,7 +403,7 @@ error_fcntl:
 }
 
 static
-void shmp_object_destroy(struct shm_object *obj)
+void shmp_object_destroy(struct shm_object *obj, int consumer)
 {
        switch (obj->type) {
        case SHM_OBJECT_SHM:
@@ -355,20 +415,46 @@ void shmp_object_destroy(struct shm_object *obj)
                        PERROR("umnmap");
                        assert(0);
                }
+
                if (obj->shm_fd_ownership) {
-                       ret = close(obj->shm_fd);
-                       if (ret) {
-                               PERROR("close");
-                               assert(0);
+                       /* Delete FDs only if called from app (not consumer). */
+                       if (!consumer) {
+                               lttng_ust_lock_fd_tracker();
+                               ret = close(obj->shm_fd);
+                               if (!ret) {
+                                       lttng_ust_delete_fd_from_tracker(obj->shm_fd);
+                               } else {
+                                       PERROR("close");
+                                       assert(0);
+                               }
+                               lttng_ust_unlock_fd_tracker();
+                       } else {
+                               ret = close(obj->shm_fd);
+                               if (ret) {
+                                       PERROR("close");
+                                       assert(0);
+                               }
                        }
                }
                for (i = 0; i < 2; i++) {
                        if (obj->wait_fd[i] < 0)
                                continue;
-                       ret = close(obj->wait_fd[i]);
-                       if (ret) {
-                               PERROR("close");
-                               assert(0);
+                       if (!consumer) {
+                               lttng_ust_lock_fd_tracker();
+                               ret = close(obj->wait_fd[i]);
+                               if (!ret) {
+                                       lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
+                               } else {
+                                       PERROR("close");
+                                       assert(0);
+                               }
+                               lttng_ust_unlock_fd_tracker();
+                       } else {
+                               ret = close(obj->wait_fd[i]);
+                               if (ret) {
+                                       PERROR("close");
+                                       assert(0);
+                               }
                        }
                }
                break;
@@ -380,10 +466,22 @@ void shmp_object_destroy(struct shm_object *obj)
                for (i = 0; i < 2; i++) {
                        if (obj->wait_fd[i] < 0)
                                continue;
-                       ret = close(obj->wait_fd[i]);
-                       if (ret) {
-                               PERROR("close");
-                               assert(0);
+                       if (!consumer) {
+                               lttng_ust_lock_fd_tracker();
+                               ret = close(obj->wait_fd[i]);
+                               if (!ret) {
+                                       lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
+                               } else {
+                                       PERROR("close");
+                                       assert(0);
+                               }
+                               lttng_ust_unlock_fd_tracker();
+                       } else {
+                               ret = close(obj->wait_fd[i]);
+                               if (ret) {
+                                       PERROR("close");
+                                       assert(0);
+                               }
                        }
                }
                free(obj->memory_map);
@@ -394,12 +492,12 @@ void shmp_object_destroy(struct shm_object *obj)
        }
 }
 
-void shm_object_table_destroy(struct shm_object_table *table)
+void shm_object_table_destroy(struct shm_object_table *table, int consumer)
 {
        int i;
 
        for (i = 0; i < table->allocated_len; i++)
-               shmp_object_destroy(&table->objects[i]);
+               shmp_object_destroy(&table->objects[i], consumer);
        free(table);
 }