Take NUMA configuration into account for UST buffer allocation

[lttng-ust.git] / libringbuffer / ring_buffer_backend.c

diff --git a/libringbuffer/ring_buffer_backend.c b/libringbuffer/ring_buffer_backend.c
index 2e688b40f4810e1a80c57b7296edafaed7e97c14..a0ef74461bd12606b9d3bd6eaa14ac2ae9f600ab 100644 (file)
--- a/libringbuffer/ring_buffer_backend.c
+++ b/libringbuffer/ring_buffer_backend.c
@@ -19,6 +19,7 @@
  */
 
 #define _GNU_SOURCE
+#define _LGPL_SOURCE
 #include <unistd.h>
 #include <urcu/arch.h>
 #include <limits.h>
@@ -117,6 +118,14 @@ int lib_ring_buffer_backend_allocate(const struct lttng_ust_lib_ring_buffer_conf
        else
                bufb->buf_rsb.id = subbuffer_id(config, 0, 1, 0);
 
+       /* Allocate subbuffer packet counter table */
+       align_shm(shmobj, __alignof__(struct lttng_ust_lib_ring_buffer_backend_counts));
+       set_shmp(bufb->buf_cnt, zalloc_shm(shmobj,
+                               sizeof(struct lttng_ust_lib_ring_buffer_backend_counts)
+                               * num_subbuf));
+       if (caa_unlikely(!shmp(handle, bufb->buf_cnt)))
+               goto free_wsb;
+
        /* Assign pages to page index */
        for (i = 0; i < num_subbuf_alloc; i++) {
                struct lttng_ust_lib_ring_buffer_backend_pages_shmp *sbp;
@@ -141,6 +150,8 @@ int lib_ring_buffer_backend_allocate(const struct lttng_ust_lib_ring_buffer_conf
        }
        return 0;
 
+free_wsb:
+       /* bufb->buf_wsb will be freed by shm teardown */
 free_array:
        /* bufb->array[i] will be freed by shm teardown */
 memory_map_error:
@@ -176,7 +187,7 @@ void lib_ring_buffer_backend_reset(struct lttng_ust_lib_ring_buffer_backend *buf
 
        chanb = &shmp(handle, bufb->chan)->backend;
        if (!chanb)
-               abort();
+               return;
        config = &chanb->config;
 
        num_subbuf_alloc = chanb->num_subbuf;
@@ -188,7 +199,7 @@ void lib_ring_buffer_backend_reset(struct lttng_ust_lib_ring_buffer_backend *buf
 
                sb = shmp_index(handle, bufb->buf_wsb, i);
                if (!sb)
-                       abort();
+                       return;
                sb->id = subbuffer_id(config, 0, 1, i);
        }
        if (chanb->extra_reader_sb)
@@ -203,10 +214,10 @@ void lib_ring_buffer_backend_reset(struct lttng_ust_lib_ring_buffer_backend *buf
 
                sbp = shmp_index(handle, bufb->array, i);
                if (!sbp)
-                       abort();
+                       return;
                pages = shmp(handle, sbp->shmp);
                if (!pages)
-                       abort();
+                       return;
                /* Don't reset mmap_offset */
                v_set(config, &pages->records_commit, 0);
                v_set(config, &pages->records_unread, 0);
@@ -323,6 +334,8 @@ int channel_backend_init(struct channel_backend *chanb,
        shmsize += sizeof(struct lttng_ust_lib_ring_buffer_backend_pages) * num_subbuf_alloc;
        shmsize += offset_align(shmsize, __alignof__(struct lttng_ust_lib_ring_buffer_backend_subbuffer));
        shmsize += sizeof(struct lttng_ust_lib_ring_buffer_backend_subbuffer) * num_subbuf;
+       shmsize += offset_align(shmsize, __alignof__(struct lttng_ust_lib_ring_buffer_backend_counts));
+       shmsize += sizeof(struct lttng_ust_lib_ring_buffer_backend_counts) * num_subbuf;
        /* Per-cpu buffer size: control (after backend) */
        shmsize += offset_align(shmsize, __alignof__(struct commit_counters_hot));
        shmsize += sizeof(struct commit_counters_hot) * num_subbuf;
@@ -338,7 +351,7 @@ int channel_backend_init(struct channel_backend *chanb,
                        struct shm_object *shmobj;
 
                        shmobj = shm_object_table_alloc(handle->table, shmsize,
-                                       SHM_OBJECT_SHM, stream_fds[i]);
+                                       SHM_OBJECT_SHM, stream_fds[i], i);
                        if (!shmobj)
                                goto end;
                        align_shm(shmobj, __alignof__(struct lttng_ust_lib_ring_buffer));
@@ -357,7 +370,7 @@ int channel_backend_init(struct channel_backend *chanb,
                struct lttng_ust_lib_ring_buffer *buf;
 
                shmobj = shm_object_table_alloc(handle->table, shmsize,
-                                       SHM_OBJECT_SHM, stream_fds[0]);
+                                       SHM_OBJECT_SHM, stream_fds[0], -1);
                if (!shmobj)
                        goto end;
                align_shm(shmobj, __alignof__(struct lttng_ust_lib_ring_buffer));
@@ -410,6 +423,7 @@ size_t lib_ring_buffer_read(struct lttng_ust_lib_ring_buffer_backend *bufb, size
        const struct lttng_ust_lib_ring_buffer_config *config;
        ssize_t orig_len;
        struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages;
+       struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages;
        unsigned long sb_bindex, id;
        void *src;
 
@@ -425,6 +439,8 @@ size_t lib_ring_buffer_read(struct lttng_ust_lib_ring_buffer_backend *bufb, size
        id = bufb->buf_rsb.id;
        sb_bindex = subbuffer_id_get_index(config, id);
        rpages = shmp_index(handle, bufb->array, sb_bindex);
+       if (!rpages)
+               return 0;
        /*
         * Underlying layer should never ask for reads across
         * subbuffers.
@@ -432,8 +448,10 @@ size_t lib_ring_buffer_read(struct lttng_ust_lib_ring_buffer_backend *bufb, size
        CHAN_WARN_ON(chanb, offset >= chanb->buf_size);
        CHAN_WARN_ON(chanb, config->mode == RING_BUFFER_OVERWRITE
                     && subbuffer_id_is_noref(config, id));
-       src = shmp_index(handle, shmp(handle, rpages->shmp)->p,
-                       offset & (chanb->subbuf_size - 1));
+       backend_pages = shmp(handle, rpages->shmp);
+       if (!backend_pages)
+               return 0;
+       src = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1));
        if (caa_unlikely(!src))
                return 0;
        memcpy(dest, src, len);
@@ -459,6 +477,7 @@ int lib_ring_buffer_read_cstr(struct lttng_ust_lib_ring_buffer_backend *bufb, si
        ssize_t string_len, orig_offset;
        char *str;
        struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages;
+       struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages;
        unsigned long sb_bindex, id;
 
        chanb = &shmp(handle, bufb->chan)->backend;
@@ -472,6 +491,8 @@ int lib_ring_buffer_read_cstr(struct lttng_ust_lib_ring_buffer_backend *bufb, si
        id = bufb->buf_rsb.id;
        sb_bindex = subbuffer_id_get_index(config, id);
        rpages = shmp_index(handle, bufb->array, sb_bindex);
+       if (!rpages)
+               return -EINVAL;
        /*
         * Underlying layer should never ask for reads across
         * subbuffers.
@@ -479,7 +500,10 @@ int lib_ring_buffer_read_cstr(struct lttng_ust_lib_ring_buffer_backend *bufb, si
        CHAN_WARN_ON(chanb, offset >= chanb->buf_size);
        CHAN_WARN_ON(chanb, config->mode == RING_BUFFER_OVERWRITE
                     && subbuffer_id_is_noref(config, id));
-       str = shmp_index(handle, shmp(handle, rpages->shmp)->p, offset & (chanb->subbuf_size - 1));
+       backend_pages = shmp(handle, rpages->shmp);
+       if (!backend_pages)
+               return -EINVAL;
+       str = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1));
        if (caa_unlikely(!str))
                return -EINVAL;
        string_len = strnlen(str, len);
@@ -506,6 +530,7 @@ void *lib_ring_buffer_read_offset_address(struct lttng_ust_lib_ring_buffer_backe
                                          struct lttng_ust_shm_handle *handle)
 {
        struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages;
+       struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages;
        struct channel_backend *chanb;
        const struct lttng_ust_lib_ring_buffer_config *config;
        unsigned long sb_bindex, id;
@@ -518,9 +543,14 @@ void *lib_ring_buffer_read_offset_address(struct lttng_ust_lib_ring_buffer_backe
        id = bufb->buf_rsb.id;
        sb_bindex = subbuffer_id_get_index(config, id);
        rpages = shmp_index(handle, bufb->array, sb_bindex);
+       if (!rpages)
+               return NULL;
        CHAN_WARN_ON(chanb, config->mode == RING_BUFFER_OVERWRITE
                     && subbuffer_id_is_noref(config, id));
-       return shmp_index(handle, shmp(handle, rpages->shmp)->p, offset & (chanb->subbuf_size - 1));
+       backend_pages = shmp(handle, rpages->shmp);
+       if (!backend_pages)
+               return NULL;
+       return shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1));
 }
 
 /**
@@ -539,6 +569,7 @@ void *lib_ring_buffer_offset_address(struct lttng_ust_lib_ring_buffer_backend *b
 {
        size_t sbidx;
        struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages;
+       struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages;
        struct channel_backend *chanb;
        const struct lttng_ust_lib_ring_buffer_config *config;
        unsigned long sb_bindex, id;
@@ -556,7 +587,12 @@ void *lib_ring_buffer_offset_address(struct lttng_ust_lib_ring_buffer_backend *b
        id = sb->id;
        sb_bindex = subbuffer_id_get_index(config, id);
        rpages = shmp_index(handle, bufb->array, sb_bindex);
+       if (!rpages)
+               return NULL;
        CHAN_WARN_ON(chanb, config->mode == RING_BUFFER_OVERWRITE
                     && subbuffer_id_is_noref(config, id));
-       return shmp_index(handle, shmp(handle, rpages->shmp)->p, offset & (chanb->subbuf_size - 1));
+       backend_pages = shmp(handle, rpages->shmp);
+       if (!backend_pages)
+               return NULL;
+       return shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1));
 }