X-Git-Url: http://git.lttng.org/?a=blobdiff_plain;f=liblttng-ust-ctl%2Fustctl.c;h=cfe2cc960a8e52c580982c09815b8ac681204ac1;hb=c0c0989ab70574e09b2f7e8b48c2da6af664a849;hp=d078af5cf2d1115b3ea4aa6f6cc6740b94732fe1;hpb=cd54f6d934faf715cbd12fa9665e94b6fe367802;p=lttng-ust.git diff --git a/liblttng-ust-ctl/ustctl.c b/liblttng-ust-ctl/ustctl.c index d078af5c..cfe2cc96 100644 --- a/liblttng-ust-ctl/ustctl.c +++ b/liblttng-ust-ctl/ustctl.c @@ -1,62 +1,118 @@ /* - * Copyright (C) 2011 - Julien Desfossez - * Mathieu Desnoyers + * SPDX-License-Identifier: GPL-2.0-only * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License only. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * Copyright (C) 2011 Julien Desfossez + * Copyright (C) 2011-2013 Mathieu Desnoyers */ -#define _GNU_SOURCE +#include #include +#include +#include +#include +#include + +#include #include #include #include -#include - +#include #include #include +#include #include "../libringbuffer/backend.h" #include "../libringbuffer/frontend.h" +#include "../liblttng-ust/wait.h" +#include "../liblttng-ust/lttng-rb-clients.h" +#include "../liblttng-ust/clock.h" +#include "../liblttng-ust/getenv.h" +#include "../liblttng-ust/lttng-tracer-core.h" -volatile enum ust_loglevel ust_loglevel; +#include "../libcounter/shm.h" +#include "../libcounter/smp.h" +#include "../libcounter/counter.h" -static -void init_object(struct lttng_ust_object_data *data) -{ - data->handle = -1; - data->shm_fd = -1; - data->wait_fd = -1; - data->memory_map_size = 0; -} +/* + * Number of milliseconds to retry before failing metadata writes on + * buffer full condition. (10 seconds) + */ +#define LTTNG_METADATA_TIMEOUT_MSEC 10000 + +/* + * Channel representation within consumer. + */ +struct ustctl_consumer_channel { + struct lttng_channel *chan; /* lttng channel buffers */ + + /* initial attributes */ + struct ustctl_consumer_channel_attr attr; + int wait_fd; /* monitor close() */ + int wakeup_fd; /* monitor close() */ +}; + +/* + * Stream representation within consumer. + */ +struct ustctl_consumer_stream { + struct lttng_ust_shm_handle *handle; /* shared-memory handle */ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *chan; + int shm_fd, wait_fd, wakeup_fd; + int cpu; + uint64_t memory_map_size; +}; + +#define USTCTL_COUNTER_ATTR_DIMENSION_MAX 8 +struct ustctl_counter_attr { + enum ustctl_counter_arithmetic arithmetic; + enum ustctl_counter_bitness bitness; + uint32_t nr_dimensions; + int64_t global_sum_step; + struct ustctl_counter_dimension dimensions[USTCTL_COUNTER_ATTR_DIMENSION_MAX]; +}; + +/* + * Counter representation within daemon. + */ +struct ustctl_daemon_counter { + struct lib_counter *counter; + const struct lttng_counter_ops *ops; + struct ustctl_counter_attr *attr; /* initial attributes */ +}; + +extern void lttng_ring_buffer_client_overwrite_init(void); +extern void lttng_ring_buffer_client_overwrite_rt_init(void); +extern void lttng_ring_buffer_client_discard_init(void); +extern void lttng_ring_buffer_client_discard_rt_init(void); +extern void lttng_ring_buffer_metadata_client_init(void); +extern void lttng_ring_buffer_client_overwrite_exit(void); +extern void lttng_ring_buffer_client_overwrite_rt_exit(void); +extern void lttng_ring_buffer_client_discard_exit(void); +extern void lttng_ring_buffer_client_discard_rt_exit(void); +extern void lttng_ring_buffer_metadata_client_exit(void); +LTTNG_HIDDEN +extern void lttng_counter_client_percpu_32_modular_init(void); +LTTNG_HIDDEN +extern void lttng_counter_client_percpu_32_modular_exit(void); +LTTNG_HIDDEN +extern void lttng_counter_client_percpu_64_modular_init(void); +LTTNG_HIDDEN +extern void lttng_counter_client_percpu_64_modular_exit(void); int ustctl_release_handle(int sock, int handle) { struct ustcomm_ust_msg lum; struct ustcomm_ust_reply lur; - int ret; - if (sock >= 0) { - memset(&lum, 0, sizeof(lum)); - lum.handle = handle; - lum.cmd = LTTNG_UST_RELEASE; - ret = ustcomm_send_app_cmd(sock, &lum, &lur); - if (ret < 0) { - return ret; - } - } - return 0; + if (sock < 0 || handle < 0) + return 0; + memset(&lum, 0, sizeof(lum)); + lum.handle = handle; + lum.cmd = LTTNG_UST_RELEASE; + return ustcomm_send_app_cmd(sock, &lum, &lur); } + /* * If sock is negative, it means we don't have to notify the other side * (e.g. application has already vanished). @@ -68,17 +124,68 @@ int ustctl_release_object(int sock, struct lttng_ust_object_data *data) if (!data) return -EINVAL; - if (data->shm_fd >= 0) { - ret = close(data->shm_fd); - if (ret < 0) { - return ret; + switch (data->type) { + case LTTNG_UST_OBJECT_TYPE_CHANNEL: + if (data->u.channel.wakeup_fd >= 0) { + ret = close(data->u.channel.wakeup_fd); + if (ret < 0) { + ret = -errno; + return ret; + } + data->u.channel.wakeup_fd = -1; } - } - if (data->wait_fd >= 0) { - ret = close(data->wait_fd); - if (ret < 0) { - return ret; + free(data->u.channel.data); + data->u.channel.data = NULL; + break; + case LTTNG_UST_OBJECT_TYPE_STREAM: + if (data->u.stream.shm_fd >= 0) { + ret = close(data->u.stream.shm_fd); + if (ret < 0) { + ret = -errno; + return ret; + } + data->u.stream.shm_fd = -1; + } + if (data->u.stream.wakeup_fd >= 0) { + ret = close(data->u.stream.wakeup_fd); + if (ret < 0) { + ret = -errno; + return ret; + } + data->u.stream.wakeup_fd = -1; + } + break; + case LTTNG_UST_OBJECT_TYPE_EVENT: + case LTTNG_UST_OBJECT_TYPE_CONTEXT: + case LTTNG_UST_OBJECT_TYPE_EVENT_NOTIFIER_GROUP: + case LTTNG_UST_OBJECT_TYPE_EVENT_NOTIFIER: + break; + case LTTNG_UST_OBJECT_TYPE_COUNTER: + free(data->u.counter.data); + data->u.counter.data = NULL; + break; + case LTTNG_UST_OBJECT_TYPE_COUNTER_GLOBAL: + if (data->u.counter_global.shm_fd >= 0) { + ret = close(data->u.counter_global.shm_fd); + if (ret < 0) { + ret = -errno; + return ret; + } + data->u.counter_global.shm_fd = -1; + } + break; + case LTTNG_UST_OBJECT_TYPE_COUNTER_CPU: + if (data->u.counter_cpu.shm_fd >= 0) { + ret = close(data->u.counter_cpu.shm_fd); + if (ret < 0) { + ret = -errno; + return ret; + } + data->u.counter_cpu.shm_fd = -1; } + break; + default: + assert(0); } return ustctl_release_handle(sock, data->handle); } @@ -99,14 +206,7 @@ int ustctl_register_done(int sock) ret = ustcomm_send_app_cmd(sock, &lum, &lur); if (ret) return ret; - if (lur.ret_code != USTCOMM_OK) { - DBG("Return code: %s", ustcomm_get_readable_code(lur.ret_code)); - goto error; - } return 0; - -error: - return -1; } /* @@ -130,200 +230,6 @@ int ustctl_create_session(int sock) return session_handle; } -/* open the metadata global channel */ -int ustctl_open_metadata(int sock, int session_handle, - struct lttng_ust_channel_attr *chops, - struct lttng_ust_object_data **_metadata_data) -{ - struct ustcomm_ust_msg lum; - struct ustcomm_ust_reply lur; - struct lttng_ust_object_data *metadata_data; - int ret, err = 0; - - if (!chops || !_metadata_data) - return -EINVAL; - - metadata_data = malloc(sizeof(*metadata_data)); - if (!metadata_data) - return -ENOMEM; - init_object(metadata_data); - /* Create metadata channel */ - memset(&lum, 0, sizeof(lum)); - lum.handle = session_handle; - lum.cmd = LTTNG_UST_METADATA; - lum.u.channel.overwrite = chops->overwrite; - lum.u.channel.subbuf_size = chops->subbuf_size; - lum.u.channel.num_subbuf = chops->num_subbuf; - lum.u.channel.switch_timer_interval = chops->switch_timer_interval; - lum.u.channel.read_timer_interval = chops->read_timer_interval; - lum.u.channel.output = chops->output; - ret = ustcomm_send_app_cmd(sock, &lum, &lur); - if (ret) { - free(metadata_data); - return ret; - } - if (lur.ret_code != USTCOMM_OK) { - free(metadata_data); - return lur.ret_code; - } - metadata_data->handle = lur.ret_val; - DBG("received metadata handle %u", metadata_data->handle); - metadata_data->memory_map_size = lur.u.channel.memory_map_size; - /* get shm fd */ - ret = ustcomm_recv_fd(sock); - if (ret < 0) - err = 1; - else - metadata_data->shm_fd = ret; - /* - * We need to get the second FD even if the first fails, because - * libust expects us to read the two FDs. - */ - /* get wait fd */ - ret = ustcomm_recv_fd(sock); - if (ret < 0) - err = 1; - else - metadata_data->wait_fd = ret; - if (err) - goto error; - *_metadata_data = metadata_data; - return 0; - -error: - (void) ustctl_release_object(sock, metadata_data); - free(metadata_data); - return -EINVAL; -} - -int ustctl_create_channel(int sock, int session_handle, - struct lttng_ust_channel_attr *chops, - struct lttng_ust_object_data **_channel_data) -{ - struct ustcomm_ust_msg lum; - struct ustcomm_ust_reply lur; - struct lttng_ust_object_data *channel_data; - int ret, err = 0; - - if (!chops || !_channel_data) - return -EINVAL; - - channel_data = malloc(sizeof(*channel_data)); - if (!channel_data) - return -ENOMEM; - init_object(channel_data); - /* Create metadata channel */ - memset(&lum, 0, sizeof(lum)); - lum.handle = session_handle; - lum.cmd = LTTNG_UST_CHANNEL; - lum.u.channel.overwrite = chops->overwrite; - lum.u.channel.subbuf_size = chops->subbuf_size; - lum.u.channel.num_subbuf = chops->num_subbuf; - lum.u.channel.switch_timer_interval = chops->switch_timer_interval; - lum.u.channel.read_timer_interval = chops->read_timer_interval; - lum.u.channel.output = chops->output; - ret = ustcomm_send_app_cmd(sock, &lum, &lur); - if (ret) { - free(channel_data); - return ret; - } - if (lur.ret_code != USTCOMM_OK) { - free(channel_data); - return lur.ret_code; - } - channel_data->handle = lur.ret_val; - DBG("received channel handle %u", channel_data->handle); - channel_data->memory_map_size = lur.u.channel.memory_map_size; - /* get shm fd */ - ret = ustcomm_recv_fd(sock); - if (ret < 0) - err = 1; - else - channel_data->shm_fd = ret; - /* - * We need to get the second FD even if the first fails, because - * libust expects us to read the two FDs. - */ - /* get wait fd */ - ret = ustcomm_recv_fd(sock); - if (ret < 0) - err = 1; - else - channel_data->wait_fd = ret; - if (err) - goto error; - *_channel_data = channel_data; - return 0; - -error: - (void) ustctl_release_object(sock, channel_data); - free(channel_data); - return -EINVAL; -} - -/* - * Return -ENOENT if no more stream is available for creation. - * Return 0 on success. - * Return negative error value on error. - */ -int ustctl_create_stream(int sock, struct lttng_ust_object_data *channel_data, - struct lttng_ust_object_data **_stream_data) -{ - struct ustcomm_ust_msg lum; - struct ustcomm_ust_reply lur; - struct lttng_ust_object_data *stream_data; - int ret, fd, err = 0; - - if (!channel_data || !_stream_data) - return -EINVAL; - - stream_data = malloc(sizeof(*stream_data)); - if (!stream_data) - return -ENOMEM; - init_object(stream_data); - memset(&lum, 0, sizeof(lum)); - lum.handle = channel_data->handle; - lum.cmd = LTTNG_UST_STREAM; - ret = ustcomm_send_app_cmd(sock, &lum, &lur); - if (ret) { - free(stream_data); - return ret; - } - if (lur.ret_code != USTCOMM_OK) { - free(stream_data); - return lur.ret_code; - } - - stream_data->handle = lur.ret_val; - DBG("received stream handle %u", stream_data->handle); - stream_data->memory_map_size = lur.u.stream.memory_map_size; - /* get shm fd */ - fd = ustcomm_recv_fd(sock); - if (fd < 0) - err = 1; - else - stream_data->shm_fd = fd; - /* - * We need to get the second FD even if the first fails, because - * libust expects us to read the two FDs. - */ - /* get wait fd */ - fd = ustcomm_recv_fd(sock); - if (fd < 0) - err = 1; - else - stream_data->wait_fd = fd; - if (err) - goto error; - *_stream_data = stream_data; - return ret; - -error: - (void) ustctl_release_object(sock, stream_data); - free(stream_data); - return -EINVAL; -} - int ustctl_create_event(int sock, struct lttng_ust_event *ev, struct lttng_ust_object_data *channel_data, struct lttng_ust_object_data **_event_data) @@ -336,10 +242,10 @@ int ustctl_create_event(int sock, struct lttng_ust_event *ev, if (!channel_data || !_event_data) return -EINVAL; - event_data = malloc(sizeof(*event_data)); + event_data = zmalloc(sizeof(*event_data)); if (!event_data) return -ENOMEM; - init_object(event_data); + event_data->type = LTTNG_UST_OBJECT_TYPE_EVENT; memset(&lum, 0, sizeof(lum)); lum.handle = channel_data->handle; lum.cmd = LTTNG_UST_EVENT; @@ -359,34 +265,86 @@ int ustctl_create_event(int sock, struct lttng_ust_event *ev, return 0; } -int ustctl_add_context(int sock, struct lttng_ust_context *ctx, +int ustctl_add_context(int sock, struct lttng_ust_context_attr *ctx, struct lttng_ust_object_data *obj_data, struct lttng_ust_object_data **_context_data) { struct ustcomm_ust_msg lum; struct ustcomm_ust_reply lur; - struct lttng_ust_object_data *context_data; + struct lttng_ust_object_data *context_data = NULL; + char *buf = NULL; + size_t len; int ret; - if (!obj_data || !_context_data) - return -EINVAL; + if (!obj_data || !_context_data) { + ret = -EINVAL; + goto end; + } - context_data = malloc(sizeof(*context_data)); - if (!context_data) - return -ENOMEM; - init_object(context_data); + context_data = zmalloc(sizeof(*context_data)); + if (!context_data) { + ret = -ENOMEM; + goto end; + } + context_data->type = LTTNG_UST_OBJECT_TYPE_CONTEXT; memset(&lum, 0, sizeof(lum)); lum.handle = obj_data->handle; lum.cmd = LTTNG_UST_CONTEXT; + lum.u.context.ctx = ctx->ctx; - ret = ustcomm_send_app_cmd(sock, &lum, &lur); - if (ret) { - free(context_data); - return ret; + switch (ctx->ctx) { + case LTTNG_UST_CONTEXT_PERF_THREAD_COUNTER: + lum.u.context.u.perf_counter = ctx->u.perf_counter; + break; + case LTTNG_UST_CONTEXT_APP_CONTEXT: + { + size_t provider_name_len = strlen( + ctx->u.app_ctx.provider_name) + 1; + size_t ctx_name_len = strlen(ctx->u.app_ctx.ctx_name) + 1; + + lum.u.context.u.app_ctx.provider_name_len = provider_name_len; + lum.u.context.u.app_ctx.ctx_name_len = ctx_name_len; + + len = provider_name_len + ctx_name_len; + buf = zmalloc(len); + if (!buf) { + ret = -ENOMEM; + goto end; + } + memcpy(buf, ctx->u.app_ctx.provider_name, + provider_name_len); + memcpy(buf + provider_name_len, ctx->u.app_ctx.ctx_name, + ctx_name_len); + break; } - context_data->handle = lur.ret_val; - DBG("received context handle %u", context_data->handle); + default: + break; + } + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) + goto end; + if (buf) { + /* send var len ctx_name */ + ret = ustcomm_send_unix_sock(sock, buf, len); + if (ret < 0) { + goto end; + } + if (ret != len) { + ret = -EINVAL; + goto end; + } + } + ret = ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); + if (ret < 0) { + goto end; + } + context_data->handle = -1; + DBG("Context created successfully"); *_context_data = context_data; + context_data = NULL; +end: + free(context_data); + free(buf); return ret; } @@ -405,23 +363,85 @@ int ustctl_set_filter(int sock, struct lttng_ust_filter_bytecode *bytecode, lum.cmd = LTTNG_UST_FILTER; lum.u.filter.data_size = bytecode->len; lum.u.filter.reloc_offset = bytecode->reloc_offset; + lum.u.filter.seqnum = bytecode->seqnum; ret = ustcomm_send_app_msg(sock, &lum); if (ret) return ret; - if (ret) { + /* send var len bytecode */ + ret = ustcomm_send_unix_sock(sock, bytecode->data, + bytecode->len); + if (ret < 0) { return ret; } + if (ret != bytecode->len) + return -EINVAL; + return ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); +} + +int ustctl_set_capture(int sock, struct lttng_ust_capture_bytecode *bytecode, + struct lttng_ust_object_data *obj_data) +{ + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + int ret; + + if (!obj_data) + return -EINVAL; + + memset(&lum, 0, sizeof(lum)); + lum.handle = obj_data->handle; + lum.cmd = LTTNG_UST_CAPTURE; + lum.u.capture.data_size = bytecode->len; + lum.u.capture.reloc_offset = bytecode->reloc_offset; + lum.u.capture.seqnum = bytecode->seqnum; + + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) + return ret; /* send var len bytecode */ ret = ustcomm_send_unix_sock(sock, bytecode->data, bytecode->len); if (ret < 0) { return ret; } - ret = ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); - if (ret) + if (ret != bytecode->len) + return -EINVAL; + return ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); +} + +int ustctl_set_exclusion(int sock, struct lttng_ust_event_exclusion *exclusion, + struct lttng_ust_object_data *obj_data) +{ + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + int ret; + + if (!obj_data) { + return -EINVAL; + } + + memset(&lum, 0, sizeof(lum)); + lum.handle = obj_data->handle; + lum.cmd = LTTNG_UST_EXCLUSION; + lum.u.exclusion.count = exclusion->count; + + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) { return ret; - return ret; + } + + /* send var len exclusion names */ + ret = ustcomm_send_unix_sock(sock, + exclusion->names, + exclusion->count * LTTNG_UST_SYM_NAME_LEN); + if (ret < 0) { + return ret; + } + if (ret != exclusion->count * LTTNG_UST_SYM_NAME_LEN) { + return -EINVAL; + } + return ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); } /* Enable event, channel and session ioctl */ @@ -480,6 +500,106 @@ int ustctl_stop_session(int sock, int handle) return ustctl_disable(sock, &obj); } +int ustctl_create_event_notifier_group(int sock, int pipe_fd, + struct lttng_ust_object_data **_event_notifier_group_data) +{ + struct lttng_ust_object_data *event_notifier_group_data; + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + ssize_t len; + int ret; + + if (!_event_notifier_group_data) + return -EINVAL; + + event_notifier_group_data = zmalloc(sizeof(*event_notifier_group_data)); + if (!event_notifier_group_data) + return -ENOMEM; + + event_notifier_group_data->type = LTTNG_UST_OBJECT_TYPE_EVENT_NOTIFIER_GROUP; + + memset(&lum, 0, sizeof(lum)); + lum.handle = LTTNG_UST_ROOT_HANDLE; + lum.cmd = LTTNG_UST_EVENT_NOTIFIER_GROUP_CREATE; + + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) + goto error; + + /* Send event_notifier notification pipe. */ + len = ustcomm_send_fds_unix_sock(sock, &pipe_fd, 1); + if (len <= 0) { + ret = len; + goto error; + } + + ret = ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); + if (ret) + goto error; + + event_notifier_group_data->handle = lur.ret_val; + DBG("received event_notifier group handle %d", event_notifier_group_data->handle); + + *_event_notifier_group_data = event_notifier_group_data; + + ret = 0; + goto end; +error: + free(event_notifier_group_data); + +end: + return ret; +} + +int ustctl_create_event_notifier(int sock, struct lttng_ust_event_notifier *event_notifier, + struct lttng_ust_object_data *event_notifier_group, + struct lttng_ust_object_data **_event_notifier_data) +{ + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + struct lttng_ust_object_data *event_notifier_data; + ssize_t len; + int ret; + + if (!event_notifier_group || !_event_notifier_data) + return -EINVAL; + + event_notifier_data = zmalloc(sizeof(*event_notifier_data)); + if (!event_notifier_data) + return -ENOMEM; + + event_notifier_data->type = LTTNG_UST_OBJECT_TYPE_EVENT_NOTIFIER; + + memset(&lum, 0, sizeof(lum)); + lum.handle = event_notifier_group->handle; + lum.cmd = LTTNG_UST_EVENT_NOTIFIER_CREATE; + lum.u.event_notifier.len = sizeof(*event_notifier); + + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) { + free(event_notifier_data); + return ret; + } + /* Send struct lttng_ust_event_notifier */ + len = ustcomm_send_unix_sock(sock, event_notifier, sizeof(*event_notifier)); + if (len != sizeof(*event_notifier)) { + if (len < 0) + return len; + else + return -EIO; + } + ret = ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); + if (ret) { + free(event_notifier_data); + return ret; + } + event_notifier_data->handle = lur.ret_val; + DBG("received event_notifier handle %u", event_notifier_data->handle); + *_event_notifier_data = event_notifier_data; + + return ret; +} + int ustctl_tracepoint_list(int sock) { struct ustcomm_ust_msg lum; @@ -554,10 +674,6 @@ int ustctl_tracepoint_field_list_get(int sock, int tp_field_list_handle, ret = ustcomm_send_app_cmd(sock, &lum, &lur); if (ret) return ret; - if (lur.ret_code != USTCOMM_OK) { - DBG("Return code: %s", ustcomm_get_readable_code(lur.ret_code)); - return -EINVAL; - } len = ustcomm_recv_unix_sock(sock, iter, sizeof(*iter)); if (len != sizeof(*iter)) { return -EINVAL; @@ -633,363 +749,2198 @@ int ustctl_sock_flush_buffer(int sock, struct lttng_ust_object_data *object) return 0; } -/* Buffer operations */ - -/* Map channel shm into process memory */ -struct lttng_ust_shm_handle *ustctl_map_channel(struct lttng_ust_object_data *chan_data) +static +int ustctl_send_channel(int sock, + enum lttng_ust_chan_type type, + void *data, + uint64_t size, + int wakeup_fd, + int send_fd_only) { - struct lttng_ust_shm_handle *handle; - struct channel *chan; - size_t chan_size; - struct lttng_ust_lib_ring_buffer_config *config; - int ret; + ssize_t len; + + if (!send_fd_only) { + /* Send mmap size */ + len = ustcomm_send_unix_sock(sock, &size, sizeof(size)); + if (len != sizeof(size)) { + if (len < 0) + return len; + else + return -EIO; + } + + /* Send channel type */ + len = ustcomm_send_unix_sock(sock, &type, sizeof(type)); + if (len != sizeof(type)) { + if (len < 0) + return len; + else + return -EIO; + } + } + + /* Send channel data */ + len = ustcomm_send_unix_sock(sock, data, size); + if (len != size) { + if (len < 0) + return len; + else + return -EIO; + } + + /* Send wakeup fd */ + len = ustcomm_send_fds_unix_sock(sock, &wakeup_fd, 1); + if (len <= 0) { + if (len < 0) + return len; + else + return -EIO; + } + return 0; +} + +static +int ustctl_send_stream(int sock, + uint32_t stream_nr, + uint64_t memory_map_size, + int shm_fd, int wakeup_fd, + int send_fd_only) +{ + ssize_t len; + int fds[2]; + + if (!send_fd_only) { + if (shm_fd < 0) { + /* finish iteration */ + uint64_t v = -1; + + len = ustcomm_send_unix_sock(sock, &v, sizeof(v)); + if (len != sizeof(v)) { + if (len < 0) + return len; + else + return -EIO; + } + return 0; + } + + /* Send mmap size */ + len = ustcomm_send_unix_sock(sock, &memory_map_size, + sizeof(memory_map_size)); + if (len != sizeof(memory_map_size)) { + if (len < 0) + return len; + else + return -EIO; + } + + /* Send stream nr */ + len = ustcomm_send_unix_sock(sock, &stream_nr, + sizeof(stream_nr)); + if (len != sizeof(stream_nr)) { + if (len < 0) + return len; + else + return -EIO; + } + } + + /* Send shm fd and wakeup fd */ + fds[0] = shm_fd; + fds[1] = wakeup_fd; + len = ustcomm_send_fds_unix_sock(sock, fds, 2); + if (len <= 0) { + if (len < 0) + return len; + else + return -EIO; + } + return 0; +} + +int ustctl_recv_channel_from_consumer(int sock, + struct lttng_ust_object_data **_channel_data) +{ + struct lttng_ust_object_data *channel_data; + ssize_t len; + int wakeup_fd; + int ret; + + channel_data = zmalloc(sizeof(*channel_data)); + if (!channel_data) { + ret = -ENOMEM; + goto error_alloc; + } + channel_data->type = LTTNG_UST_OBJECT_TYPE_CHANNEL; + channel_data->handle = -1; + + /* recv mmap size */ + len = ustcomm_recv_unix_sock(sock, &channel_data->size, + sizeof(channel_data->size)); + if (len != sizeof(channel_data->size)) { + if (len < 0) + ret = len; + else + ret = -EINVAL; + goto error; + } + + /* recv channel type */ + len = ustcomm_recv_unix_sock(sock, &channel_data->u.channel.type, + sizeof(channel_data->u.channel.type)); + if (len != sizeof(channel_data->u.channel.type)) { + if (len < 0) + ret = len; + else + ret = -EINVAL; + goto error; + } + + /* recv channel data */ + channel_data->u.channel.data = zmalloc(channel_data->size); + if (!channel_data->u.channel.data) { + ret = -ENOMEM; + goto error; + } + len = ustcomm_recv_unix_sock(sock, channel_data->u.channel.data, + channel_data->size); + if (len != channel_data->size) { + if (len < 0) + ret = len; + else + ret = -EINVAL; + goto error_recv_data; + } + /* recv wakeup fd */ + len = ustcomm_recv_fds_unix_sock(sock, &wakeup_fd, 1); + if (len <= 0) { + if (len < 0) { + ret = len; + goto error_recv_data; + } else { + ret = -EIO; + goto error_recv_data; + } + } + channel_data->u.channel.wakeup_fd = wakeup_fd; + *_channel_data = channel_data; + return 0; + +error_recv_data: + free(channel_data->u.channel.data); +error: + free(channel_data); +error_alloc: + return ret; +} + +int ustctl_recv_stream_from_consumer(int sock, + struct lttng_ust_object_data **_stream_data) +{ + struct lttng_ust_object_data *stream_data; + ssize_t len; + int ret; + int fds[2]; + + stream_data = zmalloc(sizeof(*stream_data)); + if (!stream_data) { + ret = -ENOMEM; + goto error_alloc; + } + + stream_data->type = LTTNG_UST_OBJECT_TYPE_STREAM; + stream_data->handle = -1; + + /* recv mmap size */ + len = ustcomm_recv_unix_sock(sock, &stream_data->size, + sizeof(stream_data->size)); + if (len != sizeof(stream_data->size)) { + if (len < 0) + ret = len; + else + ret = -EINVAL; + goto error; + } + if (stream_data->size == -1) { + ret = -LTTNG_UST_ERR_NOENT; + goto error; + } + + /* recv stream nr */ + len = ustcomm_recv_unix_sock(sock, &stream_data->u.stream.stream_nr, + sizeof(stream_data->u.stream.stream_nr)); + if (len != sizeof(stream_data->u.stream.stream_nr)) { + if (len < 0) + ret = len; + else + ret = -EINVAL; + goto error; + } + + /* recv shm fd and wakeup fd */ + len = ustcomm_recv_fds_unix_sock(sock, fds, 2); + if (len <= 0) { + if (len < 0) { + ret = len; + goto error; + } else { + ret = -EIO; + goto error; + } + } + stream_data->u.stream.shm_fd = fds[0]; + stream_data->u.stream.wakeup_fd = fds[1]; + *_stream_data = stream_data; + return 0; + +error: + free(stream_data); +error_alloc: + return ret; +} + +int ustctl_send_channel_to_ust(int sock, int session_handle, + struct lttng_ust_object_data *channel_data) +{ + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + int ret; + + if (!channel_data) + return -EINVAL; + + memset(&lum, 0, sizeof(lum)); + lum.handle = session_handle; + lum.cmd = LTTNG_UST_CHANNEL; + lum.u.channel.len = channel_data->size; + lum.u.channel.type = channel_data->u.channel.type; + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) + return ret; + + ret = ustctl_send_channel(sock, + channel_data->u.channel.type, + channel_data->u.channel.data, + channel_data->size, + channel_data->u.channel.wakeup_fd, + 1); + if (ret) + return ret; + ret = ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); + if (!ret) { + channel_data->handle = lur.ret_val; + } + return ret; +} + +int ustctl_send_stream_to_ust(int sock, + struct lttng_ust_object_data *channel_data, + struct lttng_ust_object_data *stream_data) +{ + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + int ret; + + memset(&lum, 0, sizeof(lum)); + lum.handle = channel_data->handle; + lum.cmd = LTTNG_UST_STREAM; + lum.u.stream.len = stream_data->size; + lum.u.stream.stream_nr = stream_data->u.stream.stream_nr; + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) + return ret; + + assert(stream_data); + assert(stream_data->type == LTTNG_UST_OBJECT_TYPE_STREAM); + + ret = ustctl_send_stream(sock, + stream_data->u.stream.stream_nr, + stream_data->size, + stream_data->u.stream.shm_fd, + stream_data->u.stream.wakeup_fd, 1); + if (ret) + return ret; + return ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); +} + +int ustctl_duplicate_ust_object_data(struct lttng_ust_object_data **dest, + struct lttng_ust_object_data *src) +{ + struct lttng_ust_object_data *obj; + int ret; + + if (src->handle != -1) { + ret = -EINVAL; + goto error; + } + + obj = zmalloc(sizeof(*obj)); + if (!obj) { + ret = -ENOMEM; + goto error; + } - if (!chan_data) + obj->type = src->type; + obj->handle = src->handle; + obj->size = src->size; + + switch (obj->type) { + case LTTNG_UST_OBJECT_TYPE_CHANNEL: + { + obj->u.channel.type = src->u.channel.type; + if (src->u.channel.wakeup_fd >= 0) { + obj->u.channel.wakeup_fd = + dup(src->u.channel.wakeup_fd); + if (obj->u.channel.wakeup_fd < 0) { + ret = errno; + goto chan_error_wakeup_fd; + } + } else { + obj->u.channel.wakeup_fd = + src->u.channel.wakeup_fd; + } + obj->u.channel.data = zmalloc(obj->size); + if (!obj->u.channel.data) { + ret = -ENOMEM; + goto chan_error_alloc; + } + memcpy(obj->u.channel.data, src->u.channel.data, obj->size); + break; + + chan_error_alloc: + if (src->u.channel.wakeup_fd >= 0) { + int closeret; + + closeret = close(obj->u.channel.wakeup_fd); + if (closeret) { + PERROR("close"); + } + } + chan_error_wakeup_fd: + goto error_type; + + } + + case LTTNG_UST_OBJECT_TYPE_STREAM: + { + obj->u.stream.stream_nr = src->u.stream.stream_nr; + if (src->u.stream.wakeup_fd >= 0) { + obj->u.stream.wakeup_fd = + dup(src->u.stream.wakeup_fd); + if (obj->u.stream.wakeup_fd < 0) { + ret = errno; + goto stream_error_wakeup_fd; + } + } else { + obj->u.stream.wakeup_fd = + src->u.stream.wakeup_fd; + } + + if (src->u.stream.shm_fd >= 0) { + obj->u.stream.shm_fd = + dup(src->u.stream.shm_fd); + if (obj->u.stream.shm_fd < 0) { + ret = errno; + goto stream_error_shm_fd; + } + } else { + obj->u.stream.shm_fd = + src->u.stream.shm_fd; + } + break; + + stream_error_shm_fd: + if (src->u.stream.wakeup_fd >= 0) { + int closeret; + + closeret = close(obj->u.stream.wakeup_fd); + if (closeret) { + PERROR("close"); + } + } + stream_error_wakeup_fd: + goto error_type; + } + + case LTTNG_UST_OBJECT_TYPE_COUNTER: + { + obj->u.counter.data = zmalloc(obj->size); + if (!obj->u.counter.data) { + ret = -ENOMEM; + goto error_type; + } + memcpy(obj->u.counter.data, src->u.counter.data, obj->size); + break; + } + + case LTTNG_UST_OBJECT_TYPE_COUNTER_GLOBAL: + { + if (src->u.counter_global.shm_fd >= 0) { + obj->u.counter_global.shm_fd = + dup(src->u.counter_global.shm_fd); + if (obj->u.counter_global.shm_fd < 0) { + ret = errno; + goto error_type; + } + } + break; + } + + case LTTNG_UST_OBJECT_TYPE_COUNTER_CPU: + { + obj->u.counter_cpu.cpu_nr = src->u.counter_cpu.cpu_nr; + if (src->u.counter_cpu.shm_fd >= 0) { + obj->u.counter_cpu.shm_fd = + dup(src->u.counter_cpu.shm_fd); + if (obj->u.counter_cpu.shm_fd < 0) { + ret = errno; + goto error_type; + } + } + break; + } + + default: + ret = -EINVAL; + goto error_type; + } + + *dest = obj; + return 0; + +error_type: + free(obj); +error: + return ret; +} + + +/* Buffer operations */ + +int ustctl_get_nr_stream_per_channel(void) +{ + return num_possible_cpus(); +} + +struct ustctl_consumer_channel * + ustctl_create_channel(struct ustctl_consumer_channel_attr *attr, + const int *stream_fds, int nr_stream_fds) +{ + struct ustctl_consumer_channel *chan; + const char *transport_name; + struct lttng_transport *transport; + + switch (attr->type) { + case LTTNG_UST_CHAN_PER_CPU: + if (attr->output == LTTNG_UST_MMAP) { + if (attr->overwrite) { + if (attr->read_timer_interval == 0) { + transport_name = "relay-overwrite-mmap"; + } else { + transport_name = "relay-overwrite-rt-mmap"; + } + } else { + if (attr->read_timer_interval == 0) { + transport_name = "relay-discard-mmap"; + } else { + transport_name = "relay-discard-rt-mmap"; + } + } + } else { + return NULL; + } + break; + case LTTNG_UST_CHAN_METADATA: + if (attr->output == LTTNG_UST_MMAP) + transport_name = "relay-metadata-mmap"; + else + return NULL; + break; + default: + transport_name = ""; + return NULL; + } + + transport = lttng_transport_find(transport_name); + if (!transport) { + DBG("LTTng transport %s not found\n", + transport_name); return NULL; + } - handle = channel_handle_create(chan_data->shm_fd, - chan_data->wait_fd, - chan_data->memory_map_size); - if (!handle) { - ERR("create handle error"); + chan = zmalloc(sizeof(*chan)); + if (!chan) return NULL; + + chan->chan = transport->ops.channel_create(transport_name, NULL, + attr->subbuf_size, attr->num_subbuf, + attr->switch_timer_interval, + attr->read_timer_interval, + attr->uuid, attr->chan_id, + stream_fds, nr_stream_fds, + attr->blocking_timeout); + if (!chan->chan) { + goto chan_error; + } + chan->chan->ops = &transport->ops; + memcpy(&chan->attr, attr, sizeof(chan->attr)); + chan->wait_fd = ustctl_channel_get_wait_fd(chan); + chan->wakeup_fd = ustctl_channel_get_wakeup_fd(chan); + return chan; + +chan_error: + free(chan); + return NULL; +} + +void ustctl_destroy_channel(struct ustctl_consumer_channel *chan) +{ + (void) ustctl_channel_close_wait_fd(chan); + (void) ustctl_channel_close_wakeup_fd(chan); + chan->chan->ops->channel_destroy(chan->chan); + free(chan); +} + +int ustctl_send_channel_to_sessiond(int sock, + struct ustctl_consumer_channel *channel) +{ + struct shm_object_table *table; + + table = channel->chan->handle->table; + if (table->size <= 0) + return -EINVAL; + return ustctl_send_channel(sock, + channel->attr.type, + table->objects[0].memory_map, + table->objects[0].memory_map_size, + channel->wakeup_fd, + 0); +} + +int ustctl_send_stream_to_sessiond(int sock, + struct ustctl_consumer_stream *stream) +{ + if (!stream) + return ustctl_send_stream(sock, -1U, -1U, -1, -1, 0); + + return ustctl_send_stream(sock, + stream->cpu, + stream->memory_map_size, + stream->shm_fd, stream->wakeup_fd, + 0); +} + +int ustctl_write_metadata_to_channel( + struct ustctl_consumer_channel *channel, + const char *metadata_str, /* NOT null-terminated */ + size_t len) /* metadata length */ +{ + struct lttng_ust_lib_ring_buffer_ctx ctx; + struct lttng_channel *chan = channel->chan; + const char *str = metadata_str; + int ret = 0, waitret; + size_t reserve_len, pos; + + for (pos = 0; pos < len; pos += reserve_len) { + reserve_len = min_t(size_t, + chan->ops->packet_avail_size(chan->chan, chan->handle), + len - pos); + lib_ring_buffer_ctx_init(&ctx, chan->chan, NULL, reserve_len, + sizeof(char), -1, chan->handle, NULL); + /* + * We don't care about metadata buffer's records lost + * count, because we always retry here. Report error if + * we need to bail out after timeout or being + * interrupted. + */ + waitret = wait_cond_interruptible_timeout( + ({ + ret = chan->ops->event_reserve(&ctx, 0); + ret != -ENOBUFS || !ret; + }), + LTTNG_METADATA_TIMEOUT_MSEC); + if (waitret == -ETIMEDOUT || waitret == -EINTR || ret) { + DBG("LTTng: Failure to write metadata to buffers (%s)\n", + waitret == -EINTR ? "interrupted" : + (ret == -ENOBUFS ? "timeout" : "I/O error")); + if (waitret == -EINTR) + ret = waitret; + goto end; + } + chan->ops->event_write(&ctx, &str[pos], reserve_len); + chan->ops->event_commit(&ctx); + } +end: + return ret; +} + +/* + * Write at most one packet in the channel. + * Returns the number of bytes written on success, < 0 on error. + */ +ssize_t ustctl_write_one_packet_to_channel( + struct ustctl_consumer_channel *channel, + const char *metadata_str, /* NOT null-terminated */ + size_t len) /* metadata length */ +{ + struct lttng_ust_lib_ring_buffer_ctx ctx; + struct lttng_channel *chan = channel->chan; + const char *str = metadata_str; + ssize_t reserve_len; + int ret; + + reserve_len = min_t(ssize_t, + chan->ops->packet_avail_size(chan->chan, chan->handle), + len); + lib_ring_buffer_ctx_init(&ctx, chan->chan, NULL, reserve_len, + sizeof(char), -1, chan->handle, NULL); + ret = chan->ops->event_reserve(&ctx, 0); + if (ret != 0) { + DBG("LTTng: event reservation failed"); + assert(ret < 0); + reserve_len = ret; + goto end; } - /* - * Set to -1, and then close the shm fd, and set the handle shm - * fd to -1 too. We don't need the shm fds after they have been - * mapped. - * The wait_fd is set to -1 in chan_data because it is now owned - * by the handle. - */ - chan_data->shm_fd = -1; - chan_data->wait_fd = -1; - - /* chan is object 0. This is hardcoded. */ - if (handle->table->objects[0].shm_fd >= 0) { - ret = close(handle->table->objects[0].shm_fd); - if (ret) { - perror("Error closing shm_fd"); - } - handle->table->objects[0].shm_fd = -1; - } - - /* - * TODO: add consistency checks to be resilient if the - * application try to feed us with incoherent channel structure - * values. - */ - chan = shmp(handle, handle->chan); - /* chan is object 0. This is hardcoded. */ - chan_size = handle->table->objects[0].allocated_len; - handle->shadow_chan = malloc(chan_size); - if (!handle->shadow_chan) { - channel_destroy(chan, handle, 1); + chan->ops->event_write(&ctx, str, reserve_len); + chan->ops->event_commit(&ctx); + +end: + return reserve_len; +} + +int ustctl_channel_close_wait_fd(struct ustctl_consumer_channel *consumer_chan) +{ + struct channel *chan; + int ret; + + chan = consumer_chan->chan->chan; + ret = ring_buffer_channel_close_wait_fd(&chan->backend.config, + chan, chan->handle); + if (!ret) + consumer_chan->wait_fd = -1; + return ret; +} + +int ustctl_channel_close_wakeup_fd(struct ustctl_consumer_channel *consumer_chan) +{ + struct channel *chan; + int ret; + + chan = consumer_chan->chan->chan; + ret = ring_buffer_channel_close_wakeup_fd(&chan->backend.config, + chan, chan->handle); + if (!ret) + consumer_chan->wakeup_fd = -1; + return ret; +} + +int ustctl_stream_close_wait_fd(struct ustctl_consumer_stream *stream) +{ + struct channel *chan; + + chan = stream->chan->chan->chan; + return ring_buffer_stream_close_wait_fd(&chan->backend.config, + chan, stream->handle, stream->cpu); +} + +int ustctl_stream_close_wakeup_fd(struct ustctl_consumer_stream *stream) +{ + struct channel *chan; + + chan = stream->chan->chan->chan; + return ring_buffer_stream_close_wakeup_fd(&chan->backend.config, + chan, stream->handle, stream->cpu); +} + +struct ustctl_consumer_stream * + ustctl_create_stream(struct ustctl_consumer_channel *channel, + int cpu) +{ + struct ustctl_consumer_stream *stream; + struct lttng_ust_shm_handle *handle; + struct channel *chan; + int shm_fd, wait_fd, wakeup_fd; + uint64_t memory_map_size; + struct lttng_ust_lib_ring_buffer *buf; + int ret; + + if (!channel) + return NULL; + handle = channel->chan->handle; + if (!handle) + return NULL; + + chan = channel->chan->chan; + buf = channel_get_ring_buffer(&chan->backend.config, + chan, cpu, handle, &shm_fd, &wait_fd, + &wakeup_fd, &memory_map_size); + if (!buf) + return NULL; + ret = lib_ring_buffer_open_read(buf, handle); + if (ret) return NULL; + + stream = zmalloc(sizeof(*stream)); + if (!stream) + goto alloc_error; + stream->handle = handle; + stream->buf = buf; + stream->chan = channel; + stream->shm_fd = shm_fd; + stream->wait_fd = wait_fd; + stream->wakeup_fd = wakeup_fd; + stream->memory_map_size = memory_map_size; + stream->cpu = cpu; + return stream; + +alloc_error: + return NULL; +} + +void ustctl_destroy_stream(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + assert(stream); + buf = stream->buf; + consumer_chan = stream->chan; + (void) ustctl_stream_close_wait_fd(stream); + (void) ustctl_stream_close_wakeup_fd(stream); + lib_ring_buffer_release_read(buf, consumer_chan->chan->handle); + free(stream); +} + +int ustctl_channel_get_wait_fd(struct ustctl_consumer_channel *chan) +{ + if (!chan) + return -EINVAL; + return shm_get_wait_fd(chan->chan->handle, + &chan->chan->handle->chan._ref); +} + +int ustctl_channel_get_wakeup_fd(struct ustctl_consumer_channel *chan) +{ + if (!chan) + return -EINVAL; + return shm_get_wakeup_fd(chan->chan->handle, + &chan->chan->handle->chan._ref); +} + +int ustctl_stream_get_wait_fd(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + return shm_get_wait_fd(consumer_chan->chan->handle, &buf->self._ref); +} + +int ustctl_stream_get_wakeup_fd(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + return shm_get_wakeup_fd(consumer_chan->chan->handle, &buf->self._ref); +} + +/* For mmap mode, readable without "get" operation */ + +void *ustctl_get_mmap_base(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + if (!stream) + return NULL; + buf = stream->buf; + consumer_chan = stream->chan; + return shmp(consumer_chan->chan->handle, buf->backend.memory_map); +} + +/* returns the length to mmap. */ +int ustctl_get_mmap_len(struct ustctl_consumer_stream *stream, + unsigned long *len) +{ + struct ustctl_consumer_channel *consumer_chan; + unsigned long mmap_buf_len; + struct channel *chan; + + if (!stream) + return -EINVAL; + consumer_chan = stream->chan; + chan = consumer_chan->chan->chan; + if (chan->backend.config.output != RING_BUFFER_MMAP) + return -EINVAL; + mmap_buf_len = chan->backend.buf_size; + if (chan->backend.extra_reader_sb) + mmap_buf_len += chan->backend.subbuf_size; + if (mmap_buf_len > INT_MAX) + return -EFBIG; + *len = mmap_buf_len; + return 0; +} + +/* returns the maximum size for sub-buffers. */ +int ustctl_get_max_subbuf_size(struct ustctl_consumer_stream *stream, + unsigned long *len) +{ + struct ustctl_consumer_channel *consumer_chan; + struct channel *chan; + + if (!stream) + return -EINVAL; + consumer_chan = stream->chan; + chan = consumer_chan->chan->chan; + *len = chan->backend.subbuf_size; + return 0; +} + +/* + * For mmap mode, operate on the current packet (between get/put or + * get_next/put_next). + */ + +/* returns the offset of the subbuffer belonging to the mmap reader. */ +int ustctl_get_mmap_read_offset(struct ustctl_consumer_stream *stream, + unsigned long *off) +{ + struct channel *chan; + unsigned long sb_bindex; + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + struct lttng_ust_lib_ring_buffer_backend_pages_shmp *barray_idx; + struct lttng_ust_lib_ring_buffer_backend_pages *pages; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + chan = consumer_chan->chan->chan; + if (chan->backend.config.output != RING_BUFFER_MMAP) + return -EINVAL; + sb_bindex = subbuffer_id_get_index(&chan->backend.config, + buf->backend.buf_rsb.id); + barray_idx = shmp_index(consumer_chan->chan->handle, buf->backend.array, + sb_bindex); + if (!barray_idx) + return -EINVAL; + pages = shmp(consumer_chan->chan->handle, barray_idx->shmp); + if (!pages) + return -EINVAL; + *off = pages->mmap_offset; + return 0; +} + +/* returns the size of the current sub-buffer, without padding (for mmap). */ +int ustctl_get_subbuf_size(struct ustctl_consumer_stream *stream, + unsigned long *len) +{ + struct ustctl_consumer_channel *consumer_chan; + struct channel *chan; + struct lttng_ust_lib_ring_buffer *buf; + + if (!stream) + return -EINVAL; + + buf = stream->buf; + consumer_chan = stream->chan; + chan = consumer_chan->chan->chan; + *len = lib_ring_buffer_get_read_data_size(&chan->backend.config, buf, + consumer_chan->chan->handle); + return 0; +} + +/* returns the size of the current sub-buffer, without padding (for mmap). */ +int ustctl_get_padded_subbuf_size(struct ustctl_consumer_stream *stream, + unsigned long *len) +{ + struct ustctl_consumer_channel *consumer_chan; + struct channel *chan; + struct lttng_ust_lib_ring_buffer *buf; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + chan = consumer_chan->chan->chan; + *len = lib_ring_buffer_get_read_data_size(&chan->backend.config, buf, + consumer_chan->chan->handle); + *len = LTTNG_UST_PAGE_ALIGN(*len); + return 0; +} + +/* Get exclusive read access to the next sub-buffer that can be read. */ +int ustctl_get_next_subbuf(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + return lib_ring_buffer_get_next_subbuf(buf, + consumer_chan->chan->handle); +} + + +/* Release exclusive sub-buffer access, move consumer forward. */ +int ustctl_put_next_subbuf(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + lib_ring_buffer_put_next_subbuf(buf, consumer_chan->chan->handle); + return 0; +} + +/* snapshot */ + +/* Get a snapshot of the current ring buffer producer and consumer positions */ +int ustctl_snapshot(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + return lib_ring_buffer_snapshot(buf, &buf->cons_snapshot, + &buf->prod_snapshot, consumer_chan->chan->handle); +} + +/* + * Get a snapshot of the current ring buffer producer and consumer positions + * even if the consumed and produced positions are contained within the same + * subbuffer. + */ +int ustctl_snapshot_sample_positions(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + return lib_ring_buffer_snapshot_sample_positions(buf, + &buf->cons_snapshot, &buf->prod_snapshot, + consumer_chan->chan->handle); +} + +/* Get the consumer position (iteration start) */ +int ustctl_snapshot_get_consumed(struct ustctl_consumer_stream *stream, + unsigned long *pos) +{ + struct lttng_ust_lib_ring_buffer *buf; + + if (!stream) + return -EINVAL; + buf = stream->buf; + *pos = buf->cons_snapshot; + return 0; +} + +/* Get the producer position (iteration end) */ +int ustctl_snapshot_get_produced(struct ustctl_consumer_stream *stream, + unsigned long *pos) +{ + struct lttng_ust_lib_ring_buffer *buf; + + if (!stream) + return -EINVAL; + buf = stream->buf; + *pos = buf->prod_snapshot; + return 0; +} + +/* Get exclusive read access to the specified sub-buffer position */ +int ustctl_get_subbuf(struct ustctl_consumer_stream *stream, + unsigned long *pos) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + return lib_ring_buffer_get_subbuf(buf, *pos, + consumer_chan->chan->handle); +} + +/* Release exclusive sub-buffer access */ +int ustctl_put_subbuf(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + if (!stream) + return -EINVAL; + buf = stream->buf; + consumer_chan = stream->chan; + lib_ring_buffer_put_subbuf(buf, consumer_chan->chan->handle); + return 0; +} + +void ustctl_flush_buffer(struct ustctl_consumer_stream *stream, + int producer_active) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + assert(stream); + buf = stream->buf; + consumer_chan = stream->chan; + lib_ring_buffer_switch_slow(buf, + producer_active ? SWITCH_ACTIVE : SWITCH_FLUSH, + consumer_chan->chan->handle); +} + +void ustctl_clear_buffer(struct ustctl_consumer_stream *stream) +{ + struct lttng_ust_lib_ring_buffer *buf; + struct ustctl_consumer_channel *consumer_chan; + + assert(stream); + buf = stream->buf; + consumer_chan = stream->chan; + lib_ring_buffer_switch_slow(buf, SWITCH_ACTIVE, + consumer_chan->chan->handle); + lib_ring_buffer_clear_reader(buf, consumer_chan->chan->handle); +} + +static +struct lttng_ust_client_lib_ring_buffer_client_cb *get_client_cb( + struct lttng_ust_lib_ring_buffer *buf, + struct lttng_ust_shm_handle *handle) +{ + struct channel *chan; + const struct lttng_ust_lib_ring_buffer_config *config; + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + + chan = shmp(handle, buf->backend.chan); + if (!chan) + return NULL; + config = &chan->backend.config; + if (!config->cb_ptr) + return NULL; + client_cb = caa_container_of(config->cb_ptr, + struct lttng_ust_client_lib_ring_buffer_client_cb, + parent); + return client_cb; +} + +int ustctl_get_timestamp_begin(struct ustctl_consumer_stream *stream, + uint64_t *timestamp_begin) +{ + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + struct lttng_ust_lib_ring_buffer *buf; + struct lttng_ust_shm_handle *handle; + + if (!stream || !timestamp_begin) + return -EINVAL; + buf = stream->buf; + handle = stream->chan->chan->handle; + client_cb = get_client_cb(buf, handle); + if (!client_cb) + return -ENOSYS; + return client_cb->timestamp_begin(buf, handle, timestamp_begin); +} + +int ustctl_get_timestamp_end(struct ustctl_consumer_stream *stream, + uint64_t *timestamp_end) +{ + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + struct lttng_ust_lib_ring_buffer *buf; + struct lttng_ust_shm_handle *handle; + + if (!stream || !timestamp_end) + return -EINVAL; + buf = stream->buf; + handle = stream->chan->chan->handle; + client_cb = get_client_cb(buf, handle); + if (!client_cb) + return -ENOSYS; + return client_cb->timestamp_end(buf, handle, timestamp_end); +} + +int ustctl_get_events_discarded(struct ustctl_consumer_stream *stream, + uint64_t *events_discarded) +{ + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + struct lttng_ust_lib_ring_buffer *buf; + struct lttng_ust_shm_handle *handle; + + if (!stream || !events_discarded) + return -EINVAL; + buf = stream->buf; + handle = stream->chan->chan->handle; + client_cb = get_client_cb(buf, handle); + if (!client_cb) + return -ENOSYS; + return client_cb->events_discarded(buf, handle, events_discarded); +} + +int ustctl_get_content_size(struct ustctl_consumer_stream *stream, + uint64_t *content_size) +{ + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + struct lttng_ust_lib_ring_buffer *buf; + struct lttng_ust_shm_handle *handle; + + if (!stream || !content_size) + return -EINVAL; + buf = stream->buf; + handle = stream->chan->chan->handle; + client_cb = get_client_cb(buf, handle); + if (!client_cb) + return -ENOSYS; + return client_cb->content_size(buf, handle, content_size); +} + +int ustctl_get_packet_size(struct ustctl_consumer_stream *stream, + uint64_t *packet_size) +{ + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + struct lttng_ust_lib_ring_buffer *buf; + struct lttng_ust_shm_handle *handle; + + if (!stream || !packet_size) + return -EINVAL; + buf = stream->buf; + handle = stream->chan->chan->handle; + client_cb = get_client_cb(buf, handle); + if (!client_cb) + return -ENOSYS; + return client_cb->packet_size(buf, handle, packet_size); +} + +int ustctl_get_stream_id(struct ustctl_consumer_stream *stream, + uint64_t *stream_id) +{ + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + struct lttng_ust_lib_ring_buffer *buf; + struct lttng_ust_shm_handle *handle; + + if (!stream || !stream_id) + return -EINVAL; + buf = stream->buf; + handle = stream->chan->chan->handle; + client_cb = get_client_cb(buf, handle); + if (!client_cb) + return -ENOSYS; + return client_cb->stream_id(buf, handle, stream_id); +} + +int ustctl_get_current_timestamp(struct ustctl_consumer_stream *stream, + uint64_t *ts) +{ + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + struct lttng_ust_lib_ring_buffer *buf; + struct lttng_ust_shm_handle *handle; + + if (!stream || !ts) + return -EINVAL; + buf = stream->buf; + handle = stream->chan->chan->handle; + client_cb = get_client_cb(buf, handle); + if (!client_cb || !client_cb->current_timestamp) + return -ENOSYS; + return client_cb->current_timestamp(buf, handle, ts); +} + +int ustctl_get_sequence_number(struct ustctl_consumer_stream *stream, + uint64_t *seq) +{ + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + struct lttng_ust_lib_ring_buffer *buf; + struct lttng_ust_shm_handle *handle; + + if (!stream || !seq) + return -EINVAL; + buf = stream->buf; + handle = stream->chan->chan->handle; + client_cb = get_client_cb(buf, handle); + if (!client_cb || !client_cb->sequence_number) + return -ENOSYS; + return client_cb->sequence_number(buf, handle, seq); +} + +int ustctl_get_instance_id(struct ustctl_consumer_stream *stream, + uint64_t *id) +{ + struct lttng_ust_client_lib_ring_buffer_client_cb *client_cb; + struct lttng_ust_lib_ring_buffer *buf; + struct lttng_ust_shm_handle *handle; + + if (!stream || !id) + return -EINVAL; + buf = stream->buf; + handle = stream->chan->chan->handle; + client_cb = get_client_cb(buf, handle); + if (!client_cb) + return -ENOSYS; + return client_cb->instance_id(buf, handle, id); +} + +#ifdef LTTNG_UST_HAVE_PERF_EVENT + +int ustctl_has_perf_counters(void) +{ + return 1; +} + +#else + +int ustctl_has_perf_counters(void) +{ + return 0; +} + +#endif + +#ifdef __linux__ +/* + * Override application pid/uid/gid with unix socket credentials. If + * the application announced a pid matching our view, it means it is + * within the same pid namespace, so expose the ppid provided by the + * application. + */ +static +int get_cred(int sock, + const struct ustctl_reg_msg *reg_msg, + uint32_t *pid, + uint32_t *ppid, + uint32_t *uid, + uint32_t *gid) +{ + struct ucred ucred; + socklen_t ucred_len = sizeof(struct ucred); + int ret; + + ret = getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &ucred_len); + if (ret) { + return -LTTNG_UST_ERR_PEERCRED; + } + DBG("Unix socket peercred [ pid: %u, uid: %u, gid: %u ], " + "application registered claiming [ pid: %u, ppid: %u, uid: %u, gid: %u ]", + ucred.pid, ucred.uid, ucred.gid, + reg_msg->pid, reg_msg->ppid, reg_msg->uid, reg_msg->gid); + if (!ucred.pid) { + ERR("Unix socket credential pid=0. Refusing application in distinct, non-nested pid namespace."); + return -LTTNG_UST_ERR_PEERCRED_PID; + } + *pid = ucred.pid; + *uid = ucred.uid; + *gid = ucred.gid; + if (ucred.pid == reg_msg->pid) { + *ppid = reg_msg->ppid; + } else { + *ppid = 0; + } + return 0; +} +#elif defined(__FreeBSD__) +#include +#include + +/* + * Override application uid/gid with unix socket credentials. Use the + * first group of the cr_groups. + * Use the pid and ppid provided by the application on registration. + */ +static +int get_cred(int sock, + const struct ustctl_reg_msg *reg_msg, + uint32_t *pid, + uint32_t *ppid, + uint32_t *uid, + uint32_t *gid) +{ + struct xucred xucred; + socklen_t xucred_len = sizeof(struct xucred); + int ret; + + ret = getsockopt(sock, SOL_SOCKET, LOCAL_PEERCRED, &xucred, &xucred_len); + if (ret) { + return -LTTNG_UST_ERR_PEERCRED; + } + if (xucred.cr_version != XUCRED_VERSION || xucred.cr_ngroups < 1) { + return -LTTNG_UST_ERR_PEERCRED; + } + DBG("Unix socket peercred [ uid: %u, gid: %u ], " + "application registered claiming [ pid: %d, ppid: %d, uid: %u, gid: %u ]", + xucred.cr_uid, xucred.cr_groups[0], + reg_msg->pid, reg_msg->ppid, reg_msg->uid, reg_msg->gid); + *pid = reg_msg->pid; + *ppid = reg_msg->ppid; + *uid = xucred.cr_uid; + *gid = xucred.cr_groups[0]; + return 0; +} +#else +#warning "Using insecure fallback: trusting user id provided by registered applications. Please consider implementing use of unix socket credentials on your platform." +static +int get_cred(int sock, + const struct ustctl_reg_msg *reg_msg, + uint32_t *pid, + uint32_t *ppid, + uint32_t *uid, + uint32_t *gid) +{ + DBG("Application registered claiming [ pid: %u, ppid: %d, uid: %u, gid: %u ]", + reg_msg->pid, reg_msg->ppid, reg_msg->uid, reg_msg->gid); + *pid = reg_msg->pid; + *ppid = reg_msg->ppid; + *uid = reg_msg->uid; + *gid = reg_msg->gid; + return 0; +} +#endif + +/* + * Returns 0 on success, negative error value on error. + */ +int ustctl_recv_reg_msg(int sock, + enum ustctl_socket_type *type, + uint32_t *major, + uint32_t *minor, + uint32_t *pid, + uint32_t *ppid, + uint32_t *uid, + uint32_t *gid, + uint32_t *bits_per_long, + uint32_t *uint8_t_alignment, + uint32_t *uint16_t_alignment, + uint32_t *uint32_t_alignment, + uint32_t *uint64_t_alignment, + uint32_t *long_alignment, + int *byte_order, + char *name) +{ + ssize_t len; + struct ustctl_reg_msg reg_msg; + + len = ustcomm_recv_unix_sock(sock, ®_msg, sizeof(reg_msg)); + if (len > 0 && len != sizeof(reg_msg)) + return -EIO; + if (len == 0) + return -EPIPE; + if (len < 0) + return len; + + if (reg_msg.magic == LTTNG_UST_COMM_MAGIC) { + *byte_order = BYTE_ORDER == BIG_ENDIAN ? + BIG_ENDIAN : LITTLE_ENDIAN; + } else if (reg_msg.magic == bswap_32(LTTNG_UST_COMM_MAGIC)) { + *byte_order = BYTE_ORDER == BIG_ENDIAN ? + LITTLE_ENDIAN : BIG_ENDIAN; + } else { + return -LTTNG_UST_ERR_INVAL_MAGIC; } - memcpy(handle->shadow_chan, chan, chan_size); - /* - * The callback pointers in the producer are invalid in the - * consumer. We need to look them up here. - */ - config = &handle->shadow_chan->backend.config; - switch (config->client_type) { - case LTTNG_CLIENT_METADATA: - memcpy(&config->cb, lttng_client_callbacks_metadata, - sizeof(config->cb)); + switch (reg_msg.socket_type) { + case 0: *type = USTCTL_SOCKET_CMD; break; - case LTTNG_CLIENT_DISCARD: - memcpy(&config->cb, lttng_client_callbacks_discard, - sizeof(config->cb)); - break; - case LTTNG_CLIENT_OVERWRITE: - memcpy(&config->cb, lttng_client_callbacks_overwrite, - sizeof(config->cb)); + case 1: *type = USTCTL_SOCKET_NOTIFY; break; default: - ERR("Unknown client type %d", config->client_type); - channel_destroy(chan, handle, 1); - return NULL; + return -LTTNG_UST_ERR_INVAL_SOCKET_TYPE; } - /* Replace the object table pointer. */ - ret = munmap(handle->table->objects[0].memory_map, - handle->table->objects[0].memory_map_size); - if (ret) { - perror("munmap"); - assert(0); + *major = reg_msg.major; + *minor = reg_msg.minor; + *bits_per_long = reg_msg.bits_per_long; + *uint8_t_alignment = reg_msg.uint8_t_alignment; + *uint16_t_alignment = reg_msg.uint16_t_alignment; + *uint32_t_alignment = reg_msg.uint32_t_alignment; + *uint64_t_alignment = reg_msg.uint64_t_alignment; + *long_alignment = reg_msg.long_alignment; + memcpy(name, reg_msg.name, LTTNG_UST_ABI_PROCNAME_LEN); + if (reg_msg.major < LTTNG_UST_ABI_MAJOR_VERSION_OLDEST_COMPATIBLE || + reg_msg.major > LTTNG_UST_ABI_MAJOR_VERSION) { + return -LTTNG_UST_ERR_UNSUP_MAJOR; } - handle->table->objects[0].memory_map = (char *) handle->shadow_chan; - handle->table->objects[0].is_shadow = 1; - return handle; + return get_cred(sock, ®_msg, pid, ppid, uid, gid); } -/* Add stream to channel shm and map its shm into process memory */ -int ustctl_add_stream(struct lttng_ust_shm_handle *handle, - struct lttng_ust_object_data *stream_data) +int ustctl_recv_notify(int sock, enum ustctl_notify_cmd *notify_cmd) { - int ret; + struct ustcomm_notify_hdr header; + ssize_t len; - if (!handle || !stream_data) + len = ustcomm_recv_unix_sock(sock, &header, sizeof(header)); + if (len > 0 && len != sizeof(header)) + return -EIO; + if (len == 0) + return -EPIPE; + if (len < 0) + return len; + switch (header.notify_cmd) { + case 0: + *notify_cmd = USTCTL_NOTIFY_CMD_EVENT; + break; + case 1: + *notify_cmd = USTCTL_NOTIFY_CMD_CHANNEL; + break; + case 2: + *notify_cmd = USTCTL_NOTIFY_CMD_ENUM; + break; + default: return -EINVAL; - - if (!stream_data->handle) - return -ENOENT; - /* map stream */ - ret = channel_handle_add_stream(handle, - stream_data->shm_fd, - stream_data->wait_fd, - stream_data->memory_map_size); - if (ret) { - ERR("add stream error\n"); - return ret; } - /* - * Set to -1 because the lttng_ust_shm_handle destruction will take care - * of closing shm_fd and wait_fd. - */ - stream_data->shm_fd = -1; - stream_data->wait_fd = -1; return 0; } -void ustctl_unmap_channel(struct lttng_ust_shm_handle *handle) -{ - struct channel *chan; - - assert(handle); - chan = shmp(handle, handle->chan); - channel_destroy(chan, handle, 1); -} - /* - * ustctl closes the shm_fd fds after mapping it. + * Returns 0 on success, negative error value on error. */ -struct lttng_ust_lib_ring_buffer *ustctl_open_stream_read(struct lttng_ust_shm_handle *handle, - int cpu) +int ustctl_recv_register_event(int sock, + int *session_objd, + int *channel_objd, + char *event_name, + int *loglevel, + char **signature, + size_t *nr_fields, + struct ustctl_field **fields, + char **model_emf_uri) { - struct channel *chan; - int *shm_fd, *wait_fd; - uint64_t *memory_map_size; - struct lttng_ust_lib_ring_buffer *buf; - int ret; + ssize_t len; + struct ustcomm_notify_event_msg msg; + size_t signature_len, fields_len, model_emf_uri_len; + char *a_sign = NULL, *a_model_emf_uri = NULL; + struct ustctl_field *a_fields = NULL; + + len = ustcomm_recv_unix_sock(sock, &msg, sizeof(msg)); + if (len > 0 && len != sizeof(msg)) + return -EIO; + if (len == 0) + return -EPIPE; + if (len < 0) + return len; + + *session_objd = msg.session_objd; + *channel_objd = msg.channel_objd; + strncpy(event_name, msg.event_name, LTTNG_UST_SYM_NAME_LEN); + event_name[LTTNG_UST_SYM_NAME_LEN - 1] = '\0'; + *loglevel = msg.loglevel; + signature_len = msg.signature_len; + fields_len = msg.fields_len; + + if (fields_len % sizeof(*a_fields) != 0) { + return -EINVAL; + } - if (!handle) - return NULL; + model_emf_uri_len = msg.model_emf_uri_len; - chan = handle->shadow_chan; - buf = channel_get_ring_buffer(&chan->backend.config, - chan, cpu, handle, &shm_fd, &wait_fd, &memory_map_size); - if (!buf) - return NULL; - ret = lib_ring_buffer_open_read(buf, handle, 1); - if (ret) - return NULL; - /* - * We can close shm_fd early, right after is has been mapped. - */ - if (*shm_fd >= 0) { - ret = close(*shm_fd); - if (ret) { - perror("Error closing shm_fd"); + /* recv signature. contains at least \0. */ + a_sign = zmalloc(signature_len); + if (!a_sign) + return -ENOMEM; + len = ustcomm_recv_unix_sock(sock, a_sign, signature_len); + if (len > 0 && len != signature_len) { + len = -EIO; + goto signature_error; + } + if (len == 0) { + len = -EPIPE; + goto signature_error; + } + if (len < 0) { + goto signature_error; + } + /* Enforce end of string */ + a_sign[signature_len - 1] = '\0'; + + /* recv fields */ + if (fields_len) { + a_fields = zmalloc(fields_len); + if (!a_fields) { + len = -ENOMEM; + goto signature_error; + } + len = ustcomm_recv_unix_sock(sock, a_fields, fields_len); + if (len > 0 && len != fields_len) { + len = -EIO; + goto fields_error; + } + if (len == 0) { + len = -EPIPE; + goto fields_error; + } + if (len < 0) { + goto fields_error; } - *shm_fd = -1; } - return buf; -} -void ustctl_close_stream_read(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf) -{ - assert(handle && buf); - lib_ring_buffer_release_read(buf, handle, 1); -} + if (model_emf_uri_len) { + /* recv model_emf_uri_len */ + a_model_emf_uri = zmalloc(model_emf_uri_len); + if (!a_model_emf_uri) { + len = -ENOMEM; + goto fields_error; + } + len = ustcomm_recv_unix_sock(sock, a_model_emf_uri, + model_emf_uri_len); + if (len > 0 && len != model_emf_uri_len) { + len = -EIO; + goto model_error; + } + if (len == 0) { + len = -EPIPE; + goto model_error; + } + if (len < 0) { + goto model_error; + } + /* Enforce end of string */ + a_model_emf_uri[model_emf_uri_len - 1] = '\0'; + } -/* For mmap mode, readable without "get" operation */ + *signature = a_sign; + *nr_fields = fields_len / sizeof(*a_fields); + *fields = a_fields; + *model_emf_uri = a_model_emf_uri; -void *ustctl_get_mmap_base(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf) -{ - if (!handle || !buf) - return NULL; - return shmp(handle, buf->backend.memory_map); + return 0; + +model_error: + free(a_model_emf_uri); +fields_error: + free(a_fields); +signature_error: + free(a_sign); + return len; } -/* returns the length to mmap. */ -int ustctl_get_mmap_len(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf, - unsigned long *len) +/* + * Returns 0 on success, negative error value on error. + */ +int ustctl_reply_register_event(int sock, + uint32_t id, + int ret_code) { - unsigned long mmap_buf_len; - struct channel *chan; + ssize_t len; + struct { + struct ustcomm_notify_hdr header; + struct ustcomm_notify_event_reply r; + } reply; + + memset(&reply, 0, sizeof(reply)); + reply.header.notify_cmd = USTCTL_NOTIFY_CMD_EVENT; + reply.r.ret_code = ret_code; + reply.r.event_id = id; + len = ustcomm_send_unix_sock(sock, &reply, sizeof(reply)); + if (len > 0 && len != sizeof(reply)) + return -EIO; + if (len < 0) + return len; + return 0; +} - if (!handle || !buf || !len) +/* + * Returns 0 on success, negative UST or system error value on error. + */ +int ustctl_recv_register_enum(int sock, + int *session_objd, + char *enum_name, + struct ustctl_enum_entry **entries, + size_t *nr_entries) +{ + ssize_t len; + struct ustcomm_notify_enum_msg msg; + size_t entries_len; + struct ustctl_enum_entry *a_entries = NULL; + + len = ustcomm_recv_unix_sock(sock, &msg, sizeof(msg)); + if (len > 0 && len != sizeof(msg)) + return -EIO; + if (len == 0) + return -EPIPE; + if (len < 0) + return len; + + *session_objd = msg.session_objd; + strncpy(enum_name, msg.enum_name, LTTNG_UST_SYM_NAME_LEN); + enum_name[LTTNG_UST_SYM_NAME_LEN - 1] = '\0'; + entries_len = msg.entries_len; + + if (entries_len % sizeof(*a_entries) != 0) { return -EINVAL; + } + + /* recv entries */ + if (entries_len) { + a_entries = zmalloc(entries_len); + if (!a_entries) + return -ENOMEM; + len = ustcomm_recv_unix_sock(sock, a_entries, entries_len); + if (len > 0 && len != entries_len) { + len = -EIO; + goto entries_error; + } + if (len == 0) { + len = -EPIPE; + goto entries_error; + } + if (len < 0) { + goto entries_error; + } + } + *nr_entries = entries_len / sizeof(*a_entries); + *entries = a_entries; - chan = handle->shadow_chan; - if (chan->backend.config.output != RING_BUFFER_MMAP) - return -EINVAL; - mmap_buf_len = chan->backend.buf_size; - if (chan->backend.extra_reader_sb) - mmap_buf_len += chan->backend.subbuf_size; - if (mmap_buf_len > INT_MAX) - return -EFBIG; - *len = mmap_buf_len; return 0; + +entries_error: + free(a_entries); + return len; } -/* returns the maximum size for sub-buffers. */ -int ustctl_get_max_subbuf_size(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf, - unsigned long *len) +/* + * Returns 0 on success, negative error value on error. + */ +int ustctl_reply_register_enum(int sock, + uint64_t id, + int ret_code) { - struct channel *chan; + ssize_t len; + struct { + struct ustcomm_notify_hdr header; + struct ustcomm_notify_enum_reply r; + } reply; + + memset(&reply, 0, sizeof(reply)); + reply.header.notify_cmd = USTCTL_NOTIFY_CMD_ENUM; + reply.r.ret_code = ret_code; + reply.r.enum_id = id; + len = ustcomm_send_unix_sock(sock, &reply, sizeof(reply)); + if (len > 0 && len != sizeof(reply)) + return -EIO; + if (len < 0) + return len; + return 0; +} - if (!handle || !buf || !len) +/* + * Returns 0 on success, negative UST or system error value on error. + */ +int ustctl_recv_register_channel(int sock, + int *session_objd, /* session descriptor (output) */ + int *channel_objd, /* channel descriptor (output) */ + size_t *nr_fields, + struct ustctl_field **fields) +{ + ssize_t len; + struct ustcomm_notify_channel_msg msg; + size_t fields_len; + struct ustctl_field *a_fields; + + len = ustcomm_recv_unix_sock(sock, &msg, sizeof(msg)); + if (len > 0 && len != sizeof(msg)) + return -EIO; + if (len == 0) + return -EPIPE; + if (len < 0) + return len; + + *session_objd = msg.session_objd; + *channel_objd = msg.channel_objd; + fields_len = msg.ctx_fields_len; + + if (fields_len % sizeof(*a_fields) != 0) { return -EINVAL; + } - chan = handle->shadow_chan; - *len = chan->backend.subbuf_size; + /* recv fields */ + if (fields_len) { + a_fields = zmalloc(fields_len); + if (!a_fields) { + len = -ENOMEM; + goto alloc_error; + } + len = ustcomm_recv_unix_sock(sock, a_fields, fields_len); + if (len > 0 && len != fields_len) { + len = -EIO; + goto fields_error; + } + if (len == 0) { + len = -EPIPE; + goto fields_error; + } + if (len < 0) { + goto fields_error; + } + *fields = a_fields; + } else { + *fields = NULL; + } + *nr_fields = fields_len / sizeof(*a_fields); return 0; + +fields_error: + free(a_fields); +alloc_error: + return len; } /* - * For mmap mode, operate on the current packet (between get/put or - * get_next/put_next). + * Returns 0 on success, negative error value on error. */ - -/* returns the offset of the subbuffer belonging to the mmap reader. */ -int ustctl_get_mmap_read_offset(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf, unsigned long *off) +int ustctl_reply_register_channel(int sock, + uint32_t chan_id, + enum ustctl_channel_header header_type, + int ret_code) { - struct channel *chan; - unsigned long sb_bindex; + ssize_t len; + struct { + struct ustcomm_notify_hdr header; + struct ustcomm_notify_channel_reply r; + } reply; + + memset(&reply, 0, sizeof(reply)); + reply.header.notify_cmd = USTCTL_NOTIFY_CMD_CHANNEL; + reply.r.ret_code = ret_code; + reply.r.chan_id = chan_id; + switch (header_type) { + case USTCTL_CHANNEL_HEADER_COMPACT: + reply.r.header_type = 1; + break; + case USTCTL_CHANNEL_HEADER_LARGE: + reply.r.header_type = 2; + break; + default: + reply.r.header_type = 0; + break; + } + len = ustcomm_send_unix_sock(sock, &reply, sizeof(reply)); + if (len > 0 && len != sizeof(reply)) + return -EIO; + if (len < 0) + return len; + return 0; +} - if (!handle || !buf || !off) - return -EINVAL; +/* Regenerate the statedump. */ +int ustctl_regenerate_statedump(int sock, int handle) +{ + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + int ret; - chan = handle->shadow_chan; - if (chan->backend.config.output != RING_BUFFER_MMAP) - return -EINVAL; - sb_bindex = subbuffer_id_get_index(&chan->backend.config, - buf->backend.buf_rsb.id); - *off = shmp(handle, shmp_index(handle, buf->backend.array, sb_bindex)->shmp)->mmap_offset; + memset(&lum, 0, sizeof(lum)); + lum.handle = handle; + lum.cmd = LTTNG_UST_SESSION_STATEDUMP; + ret = ustcomm_send_app_cmd(sock, &lum, &lur); + if (ret) + return ret; + DBG("Regenerated statedump for handle %u", handle); return 0; } -/* returns the size of the current sub-buffer, without padding (for mmap). */ -int ustctl_get_subbuf_size(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf, unsigned long *len) +/* counter operations */ + +int ustctl_get_nr_cpu_per_counter(void) { - struct channel *chan; + return lttng_counter_num_possible_cpus(); +} - if (!handle || !buf || !len) - return -EINVAL; +struct ustctl_daemon_counter * + ustctl_create_counter(size_t nr_dimensions, + const struct ustctl_counter_dimension *dimensions, + int64_t global_sum_step, + int global_counter_fd, + int nr_counter_cpu_fds, + const int *counter_cpu_fds, + enum ustctl_counter_bitness bitness, + enum ustctl_counter_arithmetic arithmetic, + uint32_t alloc_flags) +{ + const char *transport_name; + struct ustctl_daemon_counter *counter; + struct lttng_counter_transport *transport; + struct lttng_counter_dimension ust_dim[LTTNG_COUNTER_DIMENSION_MAX]; + size_t i; - chan = handle->shadow_chan; - *len = lib_ring_buffer_get_read_data_size(&chan->backend.config, buf, - handle); - return 0; + if (nr_dimensions > LTTNG_COUNTER_DIMENSION_MAX) + return NULL; + /* Currently, only per-cpu allocation is supported. */ + switch (alloc_flags) { + case USTCTL_COUNTER_ALLOC_PER_CPU: + break; + + case USTCTL_COUNTER_ALLOC_PER_CPU | USTCTL_COUNTER_ALLOC_GLOBAL: + case USTCTL_COUNTER_ALLOC_GLOBAL: + default: + return NULL; + } + switch (bitness) { + case USTCTL_COUNTER_BITNESS_32: + switch (arithmetic) { + case USTCTL_COUNTER_ARITHMETIC_MODULAR: + transport_name = "counter-per-cpu-32-modular"; + break; + case USTCTL_COUNTER_ARITHMETIC_SATURATION: + transport_name = "counter-per-cpu-32-saturation"; + break; + default: + return NULL; + } + break; + case USTCTL_COUNTER_BITNESS_64: + switch (arithmetic) { + case USTCTL_COUNTER_ARITHMETIC_MODULAR: + transport_name = "counter-per-cpu-64-modular"; + break; + case USTCTL_COUNTER_ARITHMETIC_SATURATION: + transport_name = "counter-per-cpu-64-saturation"; + break; + default: + return NULL; + } + break; + default: + return NULL; + } + + transport = lttng_counter_transport_find(transport_name); + if (!transport) { + DBG("LTTng transport %s not found\n", + transport_name); + return NULL; + } + + counter = zmalloc(sizeof(*counter)); + if (!counter) + return NULL; + counter->attr = zmalloc(sizeof(*counter->attr)); + if (!counter->attr) + goto free_counter; + counter->attr->bitness = bitness; + counter->attr->arithmetic = arithmetic; + counter->attr->nr_dimensions = nr_dimensions; + counter->attr->global_sum_step = global_sum_step; + for (i = 0; i < nr_dimensions; i++) + counter->attr->dimensions[i] = dimensions[i]; + + for (i = 0; i < nr_dimensions; i++) { + ust_dim[i].size = dimensions[i].size; + ust_dim[i].underflow_index = dimensions[i].underflow_index; + ust_dim[i].overflow_index = dimensions[i].overflow_index; + ust_dim[i].has_underflow = dimensions[i].has_underflow; + ust_dim[i].has_overflow = dimensions[i].has_overflow; + } + counter->counter = transport->ops.counter_create(nr_dimensions, + ust_dim, global_sum_step, global_counter_fd, + nr_counter_cpu_fds, counter_cpu_fds, true); + if (!counter->counter) + goto free_attr; + counter->ops = &transport->ops; + return counter; + +free_attr: + free(counter->attr); +free_counter: + free(counter); + return NULL; } -/* returns the size of the current sub-buffer, without padding (for mmap). */ -int ustctl_get_padded_subbuf_size(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf, unsigned long *len) +int ustctl_create_counter_data(struct ustctl_daemon_counter *counter, + struct lttng_ust_object_data **_counter_data) { - struct channel *chan; + struct lttng_ust_object_data *counter_data; + struct lttng_ust_counter_conf counter_conf = {0}; + size_t i; + int ret; - if (!handle || !buf || !len) + switch (counter->attr->arithmetic) { + case USTCTL_COUNTER_ARITHMETIC_MODULAR: + counter_conf.arithmetic = LTTNG_UST_COUNTER_ARITHMETIC_MODULAR; + break; + case USTCTL_COUNTER_ARITHMETIC_SATURATION: + counter_conf.arithmetic = LTTNG_UST_COUNTER_ARITHMETIC_SATURATION; + break; + default: + return -EINVAL; + } + switch (counter->attr->bitness) { + case USTCTL_COUNTER_BITNESS_32: + counter_conf.bitness = LTTNG_UST_COUNTER_BITNESS_32; + break; + case USTCTL_COUNTER_BITNESS_64: + counter_conf.bitness = LTTNG_UST_COUNTER_BITNESS_64; + break; + default: return -EINVAL; + } + counter_conf.number_dimensions = counter->attr->nr_dimensions; + counter_conf.global_sum_step = counter->attr->global_sum_step; + for (i = 0; i < counter->attr->nr_dimensions; i++) { + counter_conf.dimensions[i].size = counter->attr->dimensions[i].size; + counter_conf.dimensions[i].underflow_index = counter->attr->dimensions[i].underflow_index; + counter_conf.dimensions[i].overflow_index = counter->attr->dimensions[i].overflow_index; + counter_conf.dimensions[i].has_underflow = counter->attr->dimensions[i].has_underflow; + counter_conf.dimensions[i].has_overflow = counter->attr->dimensions[i].has_overflow; + } + + counter_data = zmalloc(sizeof(*counter_data)); + if (!counter_data) { + ret = -ENOMEM; + goto error_alloc; + } + counter_data->type = LTTNG_UST_OBJECT_TYPE_COUNTER; + counter_data->handle = -1; + + counter_data->size = sizeof(counter_conf); + counter_data->u.counter.data = zmalloc(sizeof(counter_conf)); + if (!counter_data->u.counter.data) { + ret = -ENOMEM; + goto error_alloc_data; + } + + memcpy(counter_data->u.counter.data, &counter_conf, sizeof(counter_conf)); + *_counter_data = counter_data; - chan = handle->shadow_chan; - *len = lib_ring_buffer_get_read_data_size(&chan->backend.config, buf, - handle); - *len = PAGE_ALIGN(*len); return 0; + +error_alloc_data: + free(counter_data); +error_alloc: + return ret; } -/* Get exclusive read access to the next sub-buffer that can be read. */ -int ustctl_get_next_subbuf(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf) +int ustctl_create_counter_global_data(struct ustctl_daemon_counter *counter, + struct lttng_ust_object_data **_counter_global_data) { - if (!handle || !buf) + struct lttng_ust_object_data *counter_global_data; + int ret, fd; + size_t len; + + if (lttng_counter_get_global_shm(counter->counter, &fd, &len)) return -EINVAL; + counter_global_data = zmalloc(sizeof(*counter_global_data)); + if (!counter_global_data) { + ret = -ENOMEM; + goto error_alloc; + } + counter_global_data->type = LTTNG_UST_OBJECT_TYPE_COUNTER_GLOBAL; + counter_global_data->handle = -1; + counter_global_data->size = len; + counter_global_data->u.counter_global.shm_fd = fd; + *_counter_global_data = counter_global_data; + return 0; - return lib_ring_buffer_get_next_subbuf(buf, handle); +error_alloc: + return ret; } - -/* Release exclusive sub-buffer access, move consumer forward. */ -int ustctl_put_next_subbuf(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf) +int ustctl_create_counter_cpu_data(struct ustctl_daemon_counter *counter, int cpu, + struct lttng_ust_object_data **_counter_cpu_data) { - if (!handle || !buf) - return -EINVAL; + struct lttng_ust_object_data *counter_cpu_data; + int ret, fd; + size_t len; - lib_ring_buffer_put_next_subbuf(buf, handle); + if (lttng_counter_get_cpu_shm(counter->counter, cpu, &fd, &len)) + return -EINVAL; + counter_cpu_data = zmalloc(sizeof(*counter_cpu_data)); + if (!counter_cpu_data) { + ret = -ENOMEM; + goto error_alloc; + } + counter_cpu_data->type = LTTNG_UST_OBJECT_TYPE_COUNTER_CPU; + counter_cpu_data->handle = -1; + counter_cpu_data->size = len; + counter_cpu_data->u.counter_cpu.shm_fd = fd; + counter_cpu_data->u.counter_cpu.cpu_nr = cpu; + *_counter_cpu_data = counter_cpu_data; return 0; + +error_alloc: + return ret; } -/* snapshot */ +void ustctl_destroy_counter(struct ustctl_daemon_counter *counter) +{ + counter->ops->counter_destroy(counter->counter); + free(counter->attr); + free(counter); +} -/* Get a snapshot of the current ring buffer producer and consumer positions */ -int ustctl_snapshot(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf) +int ustctl_send_counter_data_to_ust(int sock, int parent_handle, + struct lttng_ust_object_data *counter_data) { - if (!handle || !buf) + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + int ret; + size_t size; + ssize_t len; + + if (!counter_data) return -EINVAL; - return lib_ring_buffer_snapshot(buf, &buf->cons_snapshot, - &buf->prod_snapshot, handle); + size = counter_data->size; + memset(&lum, 0, sizeof(lum)); + lum.handle = parent_handle; + lum.cmd = LTTNG_UST_COUNTER; + lum.u.counter.len = size; + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) + return ret; + + /* Send counter data */ + len = ustcomm_send_unix_sock(sock, counter_data->u.counter.data, size); + if (len != size) { + if (len < 0) + return len; + else + return -EIO; + } + + ret = ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); + if (!ret) { + counter_data->handle = lur.ret_val; + } + return ret; } -/* Get the consumer position (iteration start) */ -int ustctl_snapshot_get_consumed(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf, unsigned long *pos) +int ustctl_send_counter_global_data_to_ust(int sock, + struct lttng_ust_object_data *counter_data, + struct lttng_ust_object_data *counter_global_data) { - if (!handle || !buf || !pos) + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + int ret, shm_fd[1]; + size_t size; + ssize_t len; + + if (!counter_data || !counter_global_data) return -EINVAL; - *pos = buf->cons_snapshot; - return 0; + size = counter_global_data->size; + memset(&lum, 0, sizeof(lum)); + lum.handle = counter_data->handle; /* parent handle */ + lum.cmd = LTTNG_UST_COUNTER_GLOBAL; + lum.u.counter_global.len = size; + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) + return ret; + + shm_fd[0] = counter_global_data->u.counter_global.shm_fd; + len = ustcomm_send_fds_unix_sock(sock, shm_fd, 1); + if (len <= 0) { + if (len < 0) + return len; + else + return -EIO; + } + + ret = ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); + if (!ret) { + counter_global_data->handle = lur.ret_val; + } + return ret; } -/* Get the producer position (iteration end) */ -int ustctl_snapshot_get_produced(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf, unsigned long *pos) +int ustctl_send_counter_cpu_data_to_ust(int sock, + struct lttng_ust_object_data *counter_data, + struct lttng_ust_object_data *counter_cpu_data) { - if (!handle || !buf || !pos) + struct ustcomm_ust_msg lum; + struct ustcomm_ust_reply lur; + int ret, shm_fd[1]; + size_t size; + ssize_t len; + + if (!counter_data || !counter_cpu_data) return -EINVAL; - *pos = buf->prod_snapshot; - return 0; + size = counter_cpu_data->size; + memset(&lum, 0, sizeof(lum)); + lum.handle = counter_data->handle; /* parent handle */ + lum.cmd = LTTNG_UST_COUNTER_CPU; + lum.u.counter_cpu.len = size; + lum.u.counter_cpu.cpu_nr = counter_cpu_data->u.counter_cpu.cpu_nr; + ret = ustcomm_send_app_msg(sock, &lum); + if (ret) + return ret; + + shm_fd[0] = counter_cpu_data->u.counter_global.shm_fd; + len = ustcomm_send_fds_unix_sock(sock, shm_fd, 1); + if (len <= 0) { + if (len < 0) + return len; + else + return -EIO; + } + + ret = ustcomm_recv_app_reply(sock, &lur, lum.handle, lum.cmd); + if (!ret) { + counter_cpu_data->handle = lur.ret_val; + } + return ret; } -/* Get exclusive read access to the specified sub-buffer position */ -int ustctl_get_subbuf(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf, unsigned long *pos) +int ustctl_counter_read(struct ustctl_daemon_counter *counter, + const size_t *dimension_indexes, + int cpu, int64_t *value, + bool *overflow, bool *underflow) { - if (!handle || !buf || !pos) - return -EINVAL; + return counter->ops->counter_read(counter->counter, dimension_indexes, cpu, + value, overflow, underflow); +} - return lib_ring_buffer_get_subbuf(buf, *pos, handle); +int ustctl_counter_aggregate(struct ustctl_daemon_counter *counter, + const size_t *dimension_indexes, + int64_t *value, + bool *overflow, bool *underflow) +{ + return counter->ops->counter_aggregate(counter->counter, dimension_indexes, + value, overflow, underflow); } -/* Release exclusive sub-buffer access */ -int ustctl_put_subbuf(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf) +int ustctl_counter_clear(struct ustctl_daemon_counter *counter, + const size_t *dimension_indexes) { - if (!handle || !buf) - return -EINVAL; + return counter->ops->counter_clear(counter->counter, dimension_indexes); +} - lib_ring_buffer_put_subbuf(buf, handle); - return 0; +static __attribute__((constructor)) +void ustctl_init(void) +{ + init_usterr(); + lttng_ust_getenv_init(); /* Needs init_usterr() to be completed. */ + lttng_ust_clock_init(); + lttng_ring_buffer_metadata_client_init(); + lttng_ring_buffer_client_overwrite_init(); + lttng_ring_buffer_client_overwrite_rt_init(); + lttng_ring_buffer_client_discard_init(); + lttng_ring_buffer_client_discard_rt_init(); + lttng_counter_client_percpu_32_modular_init(); + lttng_counter_client_percpu_64_modular_init(); + lib_ringbuffer_signal_init(); } -void ustctl_flush_buffer(struct lttng_ust_shm_handle *handle, - struct lttng_ust_lib_ring_buffer *buf, - int producer_active) +static __attribute__((destructor)) +void ustctl_exit(void) { - assert(handle && buf); - lib_ring_buffer_switch_slow(buf, - producer_active ? SWITCH_ACTIVE : SWITCH_FLUSH, - handle); + lttng_ring_buffer_client_discard_rt_exit(); + lttng_ring_buffer_client_discard_exit(); + lttng_ring_buffer_client_overwrite_rt_exit(); + lttng_ring_buffer_client_overwrite_exit(); + lttng_ring_buffer_metadata_client_exit(); + lttng_counter_client_percpu_32_modular_exit(); + lttng_counter_client_percpu_64_modular_exit(); }