From 309167d2a6f59d0c8cbf64eb23ba912cdea76a34 Mon Sep 17 00:00:00 2001 From: Julien Desfossez Date: Thu, 8 Aug 2013 16:54:48 -0400 Subject: [PATCH] Generate local kernel and UST indexes When the consumer creates a new trace file, it now creates as well an index file that the viewers can use to navigate in the trace. For now these indexes are only local and not sent to the relayd. This patch requires additionnal ioctl and ustctl calls implemented in lttng-modules and lttng-ust in 2.4. For now, no indexes are sent over the network and no indexes are generated on snaphots. Signed-off-by: Julien Desfossez Signed-off-by: David Goulet --- configure.ac | 1 + src/bin/lttng-consumerd/Makefile.am | 1 + src/bin/lttng-relayd/lttng-relayd.h | 1 + src/bin/lttng-relayd/main.c | 5 +- src/common/Makefile.am | 2 +- src/common/consumer-stream.c | 8 ++ src/common/consumer.c | 46 ++++++-- src/common/consumer.h | 12 ++- src/common/defaults.h | 3 + src/common/index/Makefile.am | 5 + src/common/index/index.c | 92 ++++++++++++++++ src/common/index/index.h | 30 ++++++ src/common/index/lttng-index.h | 52 +++++++++ src/common/kernel-consumer/kernel-consumer.c | 106 +++++++++++++++++-- src/common/kernel-ctl/kernel-ctl.c | 36 +++++++ src/common/kernel-ctl/kernel-ctl.h | 8 ++ src/common/kernel-ctl/kernel-ioctl.h | 13 +++ src/common/ust-consumer/ust-consumer.c | 106 +++++++++++++++++-- src/common/utils.c | 50 +++++++-- src/common/utils.h | 5 +- 20 files changed, 545 insertions(+), 37 deletions(-) create mode 100644 src/common/index/Makefile.am create mode 100644 src/common/index/index.c create mode 100644 src/common/index/index.h create mode 100644 src/common/index/lttng-index.h diff --git a/configure.ac b/configure.ac index f5f4b26d9..ac97d0eea 100644 --- a/configure.ac +++ b/configure.ac @@ -353,6 +353,7 @@ AC_CONFIG_FILES([ src/common/compat/Makefile src/common/relayd/Makefile src/common/testpoint/Makefile + src/common/index/Makefile src/lib/Makefile src/lib/lttng-ctl/Makefile src/lib/lttng-ctl/filter/Makefile diff --git a/src/bin/lttng-consumerd/Makefile.am b/src/bin/lttng-consumerd/Makefile.am index d340c68ba..a7971ade0 100644 --- a/src/bin/lttng-consumerd/Makefile.am +++ b/src/bin/lttng-consumerd/Makefile.am @@ -8,6 +8,7 @@ lttng_consumerd_LDADD = \ $(top_builddir)/src/common/libconsumer.la \ $(top_builddir)/src/common/sessiond-comm/libsessiond-comm.la \ $(top_builddir)/src/common/libcommon.la \ + $(top_builddir)/src/common/index/libindex.la \ -lrt if HAVE_LIBLTTNG_UST_CTL diff --git a/src/bin/lttng-relayd/lttng-relayd.h b/src/bin/lttng-relayd/lttng-relayd.h index 9bbafe1dd..61db23a77 100644 --- a/src/bin/lttng-relayd/lttng-relayd.h +++ b/src/bin/lttng-relayd/lttng-relayd.h @@ -60,6 +60,7 @@ struct relay_stream { struct relay_session *session; struct rcu_head rcu_node; int fd; + int index_fd; char *path_name; char *channel_name; diff --git a/src/bin/lttng-relayd/main.c b/src/bin/lttng-relayd/main.c index 53f1b49ce..bb038a670 100644 --- a/src/bin/lttng-relayd/main.c +++ b/src/bin/lttng-relayd/main.c @@ -868,7 +868,7 @@ int relay_add_stream(struct lttcomm_relayd_hdr *recv_hdr, * uses its own credentials for the stream files. */ ret = utils_create_stream_file(stream->path_name, stream->channel_name, - stream->tracefile_size, 0, -1, -1); + stream->tracefile_size, 0, -1, -1, NULL); if (ret < 0) { ERR("Create output file"); goto end; @@ -1626,7 +1626,8 @@ int relay_process_data(struct relay_command *cmd, struct lttng_ht *streams_ht) ret = utils_rotate_stream_file(stream->path_name, stream->channel_name, stream->tracefile_size, stream->tracefile_count, -1, -1, - stream->fd, &(stream->tracefile_count_current)); + stream->fd, &(stream->tracefile_count_current), + &stream->fd); if (ret < 0) { ERR("Rotating output file"); goto end; diff --git a/src/common/Makefile.am b/src/common/Makefile.am index aedbc22bd..8454b5a51 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -1,7 +1,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src SUBDIRS = compat hashtable kernel-ctl sessiond-comm relayd \ - kernel-consumer ust-consumer testpoint + kernel-consumer ust-consumer testpoint index AM_CFLAGS = -fno-strict-aliasing diff --git a/src/common/consumer-stream.c b/src/common/consumer-stream.c index 717e0a735..2bb5ce7e8 100644 --- a/src/common/consumer-stream.c +++ b/src/common/consumer-stream.c @@ -135,6 +135,14 @@ void consumer_stream_close(struct lttng_consumer_stream *stream) stream->out_fd = -1; } + if (stream->index_fd >= 0) { + ret = close(stream->index_fd); + if (ret) { + PERROR("close stream index_fd"); + } + stream->index_fd = -1; + } + /* Check and cleanup relayd if needed. */ rcu_read_lock(); relayd = consumer_find_relayd(stream->net_seq_idx); diff --git a/src/common/consumer.c b/src/common/consumer.c index da24f81c5..2c2b79cf0 100644 --- a/src/common/consumer.c +++ b/src/common/consumer.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -506,6 +507,7 @@ struct lttng_consumer_stream *consumer_allocate_stream(uint64_t channel_key, stream->session_id = session_id; stream->monitor = monitor; stream->endpoint_status = CONSUMER_ENDPOINT_ACTIVE; + stream->index_fd = -1; pthread_mutex_init(&stream->lock, NULL); /* If channel is the metadata, flag this stream as metadata. */ @@ -1317,7 +1319,8 @@ end: ssize_t lttng_consumer_on_read_subbuffer_mmap( struct lttng_consumer_local_data *ctx, struct lttng_consumer_stream *stream, unsigned long len, - unsigned long padding) + unsigned long padding, + struct lttng_packet_index *index) { unsigned long mmap_offset; void *mmap_base; @@ -1424,18 +1427,34 @@ ssize_t lttng_consumer_on_read_subbuffer_mmap( ret = utils_rotate_stream_file(stream->chan->pathname, stream->name, stream->chan->tracefile_size, stream->chan->tracefile_count, stream->uid, stream->gid, - stream->out_fd, &(stream->tracefile_count_current)); + stream->out_fd, &(stream->tracefile_count_current), + &stream->out_fd); if (ret < 0) { ERR("Rotating output file"); goto end; } - outfd = stream->out_fd = ret; + outfd = stream->out_fd; + + if (stream->index_fd >= 0) { + ret = index_create_file(stream->chan->pathname, + stream->name, stream->uid, stream->gid, + stream->chan->tracefile_size, + stream->tracefile_count_current); + if (ret < 0) { + goto end; + } + stream->index_fd = ret; + } + /* Reset current size because we just perform a rotation. */ stream->tracefile_size_current = 0; stream->out_fd_offset = 0; orig_offset = 0; } stream->tracefile_size_current += len; + if (index) { + index->offset = htobe64(stream->out_fd_offset); + } } while (len > 0) { @@ -1510,7 +1529,8 @@ end: ssize_t lttng_consumer_on_read_subbuffer_splice( struct lttng_consumer_local_data *ctx, struct lttng_consumer_stream *stream, unsigned long len, - unsigned long padding) + unsigned long padding, + struct lttng_packet_index *index) { ssize_t ret = 0, written = 0, ret_splice = 0; loff_t offset = 0; @@ -1610,18 +1630,32 @@ ssize_t lttng_consumer_on_read_subbuffer_splice( ret = utils_rotate_stream_file(stream->chan->pathname, stream->name, stream->chan->tracefile_size, stream->chan->tracefile_count, stream->uid, stream->gid, - stream->out_fd, &(stream->tracefile_count_current)); + stream->out_fd, &(stream->tracefile_count_current), + &stream->out_fd); if (ret < 0) { ERR("Rotating output file"); goto end; } - outfd = stream->out_fd = ret; + outfd = stream->out_fd; + + if (stream->index_fd >= 0) { + ret = index_create_file(stream->chan->pathname, + stream->name, stream->uid, stream->gid, + stream->chan->tracefile_size, + stream->tracefile_count_current); + if (ret < 0) { + goto end; + } + stream->index_fd = ret; + } + /* Reset current size because we just perform a rotation. */ stream->tracefile_size_current = 0; stream->out_fd_offset = 0; orig_offset = 0; } stream->tracefile_size_current += len; + index->offset = htobe64(stream->out_fd_offset); } while (len > 0) { diff --git a/src/common/consumer.h b/src/common/consumer.h index 91f6b5ca1..4831ce898 100644 --- a/src/common/consumer.h +++ b/src/common/consumer.h @@ -32,6 +32,7 @@ #include #include #include +#include /* Commands for consumer */ enum lttng_consumer_command { @@ -321,6 +322,10 @@ struct lttng_consumer_stream { * to the channel. */ uint64_t ust_metadata_pushed; + /* + * FD of the index file for this stream. + */ + int index_fd; }; /* @@ -590,11 +595,13 @@ void lttng_consumer_destroy(struct lttng_consumer_local_data *ctx); ssize_t lttng_consumer_on_read_subbuffer_mmap( struct lttng_consumer_local_data *ctx, struct lttng_consumer_stream *stream, unsigned long len, - unsigned long padding); + unsigned long padding, + struct lttng_packet_index *index); ssize_t lttng_consumer_on_read_subbuffer_splice( struct lttng_consumer_local_data *ctx, struct lttng_consumer_stream *stream, unsigned long len, - unsigned long padding); + unsigned long padding, + struct lttng_packet_index *index); int lttng_consumer_take_snapshot(struct lttng_consumer_stream *stream); int lttng_consumer_get_produced_snapshot(struct lttng_consumer_stream *stream, unsigned long *pos); @@ -627,5 +634,6 @@ int consumer_add_data_stream(struct lttng_consumer_stream *stream); void consumer_del_stream_for_data(struct lttng_consumer_stream *stream); int consumer_add_metadata_stream(struct lttng_consumer_stream *stream); void consumer_del_stream_for_metadata(struct lttng_consumer_stream *stream); +int consumer_create_index_file(struct lttng_consumer_stream *stream); #endif /* LIB_CONSUMER_H */ diff --git a/src/common/defaults.h b/src/common/defaults.h index c115491ec..d1e4adfcd 100644 --- a/src/common/defaults.h +++ b/src/common/defaults.h @@ -216,6 +216,9 @@ #define DEFAULT_SNAPSHOT_NAME "snapshot" #define DEFAULT_SNAPSHOT_MAX_SIZE 0 /* Unlimited. */ +/* Suffix of an index file. */ +#define DEFAULT_INDEX_FILE_SUFFIX ".idx" + extern size_t default_channel_subbuf_size; extern size_t default_metadata_subbuf_size; extern size_t default_ust_pid_channel_subbuf_size; diff --git a/src/common/index/Makefile.am b/src/common/index/Makefile.am new file mode 100644 index 000000000..104d99c50 --- /dev/null +++ b/src/common/index/Makefile.am @@ -0,0 +1,5 @@ +AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src + +noinst_LTLIBRARIES = libindex.la + +libindex_la_SOURCES = index.c index.h lttng-index.h diff --git a/src/common/index/index.c b/src/common/index/index.c new file mode 100644 index 000000000..3d22ca61c --- /dev/null +++ b/src/common/index/index.c @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2013 - Julien Desfossez + * David Goulet + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License, version 2 only, as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include + +#include +#include +#include + +#include "index.h" + +/* + * Create the index file associated with a trace file. + * + * Return fd on success, a negative value on error. + */ +int index_create_file(char *path_name, char *stream_name, int uid, int gid, + uint64_t size, uint64_t count) +{ + int ret, fd = -1; + struct lttng_packet_index_file_hdr hdr; + + ret = utils_create_stream_file(path_name, stream_name, size, count, uid, + gid, DEFAULT_INDEX_FILE_SUFFIX); + if (ret < 0) { + goto error; + } + fd = ret; + + memcpy(hdr.magic, INDEX_MAGIC, sizeof(hdr.magic)); + hdr.index_major = htobe32(INDEX_MAJOR); + hdr.index_minor = htobe32(INDEX_MINOR); + + do { + ret = write(fd, &hdr, sizeof(hdr)); + } while (ret < 0 && errno == EINTR); + if (ret < 0) { + PERROR("write index header"); + goto error; + } + + return fd; + +error: + if (fd >= 0) { + int close_ret; + + close_ret = close(fd); + if (close_ret < 0) { + PERROR("close index fd"); + } + } + return ret; +} + +/* + * Write index values to the given fd of size len. + * + * Return 0 on success or else a negative value on error. + */ +int index_write(int fd, struct lttng_packet_index *index, size_t len) +{ + int ret; + + assert(fd >= 0); + assert(index); + + do { + ret = write(fd, index, len); + } while (ret < 0 && errno == EINTR); + if (ret < 0) { + PERROR("writing index file"); + } + + return ret; +} diff --git a/src/common/index/index.h b/src/common/index/index.h new file mode 100644 index 000000000..58f2ac70a --- /dev/null +++ b/src/common/index/index.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2013 - Julien Desfossez + * David Goulet + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License, version 2 only, as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _INDEX_H +#define _INDEX_H + +#include + +#include "lttng-index.h" + +int index_create_file(char *path_name, char *stream_name, int uid, int gid, + uint64_t size, uint64_t count); +int index_write(int fd, struct lttng_packet_index *index, size_t len); + +#endif /* _INDEX_H */ diff --git a/src/common/index/lttng-index.h b/src/common/index/lttng-index.h new file mode 100644 index 000000000..d85b0edba --- /dev/null +++ b/src/common/index/lttng-index.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2013 - Julien Desfossez + * David Goulet + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef LTTNG_INDEX_H +#define LTTNG_INDEX_H + +#include + +#define INDEX_MAGIC "CTFIDX" +#define INDEX_MAJOR 1 +#define INDEX_MINOR 0 + +/* + * Header at the beginning of each index file. + * All integer fields are stored in big endian. + */ +struct lttng_packet_index_file_hdr { + char magic[6]; + uint32_t index_major; + uint32_t index_minor; +} __attribute__((__packed__)); + +/* + * Packet index generated for each trace packet store in a trace file. + * All integer fields are stored in big endian. + */ +struct lttng_packet_index { + uint64_t offset; /* offset of the packet in the file, in bytes */ + uint64_t packet_size; /* packet size, in bits */ + uint64_t content_size; /* content size, in bits */ + uint64_t timestamp_begin; + uint64_t timestamp_end; + uint64_t events_discarded; + uint64_t stream_id; +} __attribute__((__packed__)); + +#endif /* LTTNG_INDEX_H */ diff --git a/src/common/kernel-consumer/kernel-consumer.c b/src/common/kernel-consumer/kernel-consumer.c index bfec4d2f0..09ccda329 100644 --- a/src/common/kernel-consumer/kernel-consumer.c +++ b/src/common/kernel-consumer/kernel-consumer.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "kernel-consumer.h" @@ -158,7 +159,7 @@ int lttng_kconsumer_snapshot_channel(uint64_t key, char *path, ret = utils_create_stream_file(path, stream->name, stream->chan->tracefile_size, stream->tracefile_count_current, - stream->uid, stream->gid); + stream->uid, stream->gid, NULL); if (ret < 0) { ERR("utils_create_stream_file"); goto end_unlock; @@ -248,7 +249,7 @@ int lttng_kconsumer_snapshot_channel(uint64_t key, char *path, } read_len = lttng_consumer_on_read_subbuffer_mmap(ctx, stream, len, - padded_len - len); + padded_len - len, NULL); /* * We write the padded len in local tracefiles but the data len * when using a relay. Display the error but continue processing @@ -352,7 +353,7 @@ int lttng_kconsumer_snapshot_metadata(uint64_t key, char *path, ret = utils_create_stream_file(path, metadata_stream->name, metadata_stream->chan->tracefile_size, metadata_stream->tracefile_count_current, - metadata_stream->uid, metadata_stream->gid); + metadata_stream->uid, metadata_stream->gid, NULL); if (ret < 0) { goto error; } @@ -843,6 +844,61 @@ error_fatal: return -1; } +/* + * Populate index values of a kernel stream. Values are set in big endian order. + * + * Return 0 on success or else a negative value. + */ +static int get_index_values(struct lttng_packet_index *index, int infd) +{ + int ret; + + ret = kernctl_get_timestamp_begin(infd, &index->timestamp_begin); + if (ret < 0) { + PERROR("kernctl_get_timestamp_begin"); + goto error; + } + index->timestamp_begin = htobe64(index->timestamp_begin); + + ret = kernctl_get_timestamp_end(infd, &index->timestamp_end); + if (ret < 0) { + PERROR("kernctl_get_timestamp_end"); + goto error; + } + index->timestamp_end = htobe64(index->timestamp_end); + + ret = kernctl_get_events_discarded(infd, &index->events_discarded); + if (ret < 0) { + PERROR("kernctl_get_events_discarded"); + goto error; + } + index->events_discarded = htobe64(index->events_discarded); + + ret = kernctl_get_content_size(infd, &index->content_size); + if (ret < 0) { + PERROR("kernctl_get_content_size"); + goto error; + } + index->content_size = htobe64(index->content_size); + + ret = kernctl_get_packet_size(infd, &index->packet_size); + if (ret < 0) { + PERROR("kernctl_get_packet_size"); + goto error; + } + index->packet_size = htobe64(index->packet_size); + + ret = kernctl_get_stream_id(infd, &index->stream_id); + if (ret < 0) { + PERROR("kernctl_get_stream_id"); + goto error; + } + index->stream_id = htobe64(index->stream_id); + +error: + return ret; +} + /* * Consume data on a file descriptor and write it on a trace file. */ @@ -850,11 +906,18 @@ ssize_t lttng_kconsumer_read_subbuffer(struct lttng_consumer_stream *stream, struct lttng_consumer_local_data *ctx) { unsigned long len, subbuf_size, padding; - int err; + int err, write_index = 0; ssize_t ret = 0; int infd = stream->wait_fd; + struct lttng_packet_index index; DBG("In read_subbuffer (infd : %d)", infd); + + /* Indicate that for this stream we have to write the index. */ + if (stream->index_fd >= 0) { + write_index = 1; + } + /* Get the next subbuffer */ err = kernctl_get_next_subbuf(infd); if (err != 0) { @@ -878,6 +941,13 @@ ssize_t lttng_kconsumer_read_subbuffer(struct lttng_consumer_stream *stream, goto end; } + if (!stream->metadata_flag && write_index) { + ret = get_index_values(&index, infd); + if (ret < 0) { + goto end; + } + } + switch (stream->chan->output) { case CONSUMER_CHANNEL_SPLICE: /* @@ -890,7 +960,7 @@ ssize_t lttng_kconsumer_read_subbuffer(struct lttng_consumer_stream *stream, /* splice the subbuffer to the tracefile */ ret = lttng_consumer_on_read_subbuffer_splice(ctx, stream, subbuf_size, - padding); + padding, &index); /* * XXX: Splice does not support network streaming so the return value * is simply checked against subbuf_size and not like the mmap() op. @@ -902,6 +972,7 @@ ssize_t lttng_kconsumer_read_subbuffer(struct lttng_consumer_stream *stream, */ ERR("Error splicing to tracefile (ret: %zd != len: %lu)", ret, subbuf_size); + write_index = 0; } break; case CONSUMER_CHANNEL_MMAP: @@ -920,7 +991,7 @@ ssize_t lttng_kconsumer_read_subbuffer(struct lttng_consumer_stream *stream, /* write the subbuffer to the tracefile */ ret = lttng_consumer_on_read_subbuffer_mmap(ctx, stream, subbuf_size, - padding); + padding, &index); /* * The mmap operation should write subbuf_size amount of data when * network streaming or the full padding (len) size when we are _not_ @@ -935,6 +1006,7 @@ ssize_t lttng_kconsumer_read_subbuffer(struct lttng_consumer_stream *stream, ERR("Error writing to tracefile " "(ret: %zd != len: %lu != subbuf_size: %lu)", ret, len, subbuf_size); + write_index = 0; } break; default: @@ -954,6 +1026,15 @@ ssize_t lttng_kconsumer_read_subbuffer(struct lttng_consumer_stream *stream, goto end; } + /* Write index if needed. */ + if (write_index) { + err = index_write(stream->index_fd, &index, sizeof(index)); + if (err < 0) { + ret = -1; + goto end; + } + } + end: return ret; } @@ -971,12 +1052,23 @@ int lttng_kconsumer_on_recv_stream(struct lttng_consumer_stream *stream) if (stream->net_seq_idx == (uint64_t) -1ULL && stream->chan->monitor) { ret = utils_create_stream_file(stream->chan->pathname, stream->name, stream->chan->tracefile_size, stream->tracefile_count_current, - stream->uid, stream->gid); + stream->uid, stream->gid, NULL); if (ret < 0) { goto error; } stream->out_fd = ret; stream->tracefile_size_current = 0; + + if (!stream->metadata_flag) { + ret = index_create_file(stream->chan->pathname, + stream->name, stream->uid, stream->gid, + stream->chan->tracefile_size, + stream->tracefile_count_current); + if (ret < 0) { + goto error; + } + stream->index_fd = ret; + } } if (stream->output == LTTNG_EVENT_MMAP) { diff --git a/src/common/kernel-ctl/kernel-ctl.c b/src/common/kernel-ctl/kernel-ctl.c index d850f38c0..495301e5f 100644 --- a/src/common/kernel-ctl/kernel-ctl.c +++ b/src/common/kernel-ctl/kernel-ctl.c @@ -390,3 +390,39 @@ int kernctl_put_subbuf(int fd) { return ioctl(fd, RING_BUFFER_PUT_SUBBUF); } + +/* Returns the timestamp begin of the current sub-buffer. */ +int kernctl_get_timestamp_begin(int fd, uint64_t *timestamp_begin) +{ + return ioctl(fd, LTTNG_RING_BUFFER_GET_TIMESTAMP_BEGIN, timestamp_begin); +} + +/* Returns the timestamp end of the current sub-buffer. */ +int kernctl_get_timestamp_end(int fd, uint64_t *timestamp_end) +{ + return ioctl(fd, LTTNG_RING_BUFFER_GET_TIMESTAMP_END, timestamp_end); +} + +/* Returns the number of discarded events in the current sub-buffer. */ +int kernctl_get_events_discarded(int fd, uint64_t *events_discarded) +{ + return ioctl(fd, LTTNG_RING_BUFFER_GET_EVENTS_DISCARDED, events_discarded); +} + +/* Returns the content size in the current sub-buffer. */ +int kernctl_get_content_size(int fd, uint64_t *content_size) +{ + return ioctl(fd, LTTNG_RING_BUFFER_GET_CONTENT_SIZE, content_size); +} + +/* Returns the packet size in the current sub-buffer. */ +int kernctl_get_packet_size(int fd, uint64_t *packet_size) +{ + return ioctl(fd, LTTNG_RING_BUFFER_GET_PACKET_SIZE, packet_size); +} + +/* Returns the stream id of the current sub-buffer. */ +int kernctl_get_stream_id(int fd, uint64_t *stream_id) +{ + return ioctl(fd, LTTNG_RING_BUFFER_GET_STREAM_ID, stream_id); +} diff --git a/src/common/kernel-ctl/kernel-ctl.h b/src/common/kernel-ctl/kernel-ctl.h index ea2aa5819..badf609a0 100644 --- a/src/common/kernel-ctl/kernel-ctl.h +++ b/src/common/kernel-ctl/kernel-ctl.h @@ -67,4 +67,12 @@ int kernctl_put_subbuf(int fd); int kernctl_buffer_flush(int fd); +/* index */ +int kernctl_get_timestamp_begin(int fd, uint64_t *timestamp_begin); +int kernctl_get_timestamp_end(int fd, uint64_t *timestamp_end); +int kernctl_get_events_discarded(int fd, uint64_t *events_discarded); +int kernctl_get_content_size(int fd, uint64_t *content_size); +int kernctl_get_packet_size(int fd, uint64_t *packet_size); +int kernctl_get_stream_id(int fd, uint64_t *stream_id); + #endif /* _LTTNG_KERNEL_CTL_H */ diff --git a/src/common/kernel-ctl/kernel-ioctl.h b/src/common/kernel-ctl/kernel-ioctl.h index 75d6da0cc..1a3b16967 100644 --- a/src/common/kernel-ctl/kernel-ioctl.h +++ b/src/common/kernel-ctl/kernel-ioctl.h @@ -47,6 +47,19 @@ /* flush the current sub-buffer */ #define RING_BUFFER_FLUSH _IO(0xF6, 0x0C) +/* returns the timestamp begin of the current sub-buffer */ +#define LTTNG_RING_BUFFER_GET_TIMESTAMP_BEGIN _IOR(0xF6, 0x20, uint64_t) +/* returns the timestamp end of the current sub-buffer */ +#define LTTNG_RING_BUFFER_GET_TIMESTAMP_END _IOR(0xF6, 0x21, uint64_t) +/* returns the number of events discarded */ +#define LTTNG_RING_BUFFER_GET_EVENTS_DISCARDED _IOR(0xF6, 0x22, uint64_t) +/* returns the packet payload size */ +#define LTTNG_RING_BUFFER_GET_CONTENT_SIZE _IOR(0xF6, 0x23, uint64_t) +/* returns the actual packet size */ +#define LTTNG_RING_BUFFER_GET_PACKET_SIZE _IOR(0xF6, 0x24, uint64_t) +/* returns the stream id */ +#define LTTNG_RING_BUFFER_GET_STREAM_ID _IOR(0xF6, 0x25, uint64_t) + /* Old ABI (without support for 32/64 bits compat) */ /* LTTng file descriptor ioctl */ #define LTTNG_KERNEL_OLD_SESSION _IO(0xF6, 0x40) diff --git a/src/common/ust-consumer/ust-consumer.c b/src/common/ust-consumer/ust-consumer.c index 6a692b9ba..c794b93f8 100644 --- a/src/common/ust-consumer/ust-consumer.c +++ b/src/common/ust-consumer/ust-consumer.c @@ -40,6 +40,7 @@ #include #include #include +#include #include "ust-consumer.h" @@ -825,7 +826,7 @@ static int snapshot_metadata(uint64_t key, char *path, uint64_t relayd_id, ret = utils_create_stream_file(path, metadata_stream->name, metadata_stream->chan->tracefile_size, metadata_stream->tracefile_count_current, - metadata_stream->uid, metadata_stream->gid); + metadata_stream->uid, metadata_stream->gid, NULL); if (ret < 0) { goto error_stream; } @@ -905,7 +906,7 @@ static int snapshot_channel(uint64_t key, char *path, uint64_t relayd_id, ret = utils_create_stream_file(path, stream->name, stream->chan->tracefile_size, stream->tracefile_count_current, - stream->uid, stream->gid); + stream->uid, stream->gid, NULL); if (ret < 0) { goto error_unlock; } @@ -975,7 +976,7 @@ static int snapshot_channel(uint64_t key, char *path, uint64_t relayd_id, } read_len = lttng_consumer_on_read_subbuffer_mmap(ctx, stream, len, - padded_len - len); + padded_len - len, NULL); if (use_relayd) { if (read_len != len) { ret = -EPERM; @@ -1600,14 +1601,72 @@ void lttng_ustconsumer_del_stream(struct lttng_consumer_stream *stream) ustctl_destroy_stream(stream->ustream); } +/* + * Populate index values of a UST stream. Values are set in big endian order. + * + * Return 0 on success or else a negative value. + */ +static int get_index_values(struct lttng_packet_index *index, + struct ustctl_consumer_stream *ustream) +{ + int ret; + + ret = ustctl_get_timestamp_begin(ustream, &index->timestamp_begin); + if (ret < 0) { + PERROR("ustctl_get_timestamp_begin"); + goto error; + } + index->timestamp_begin = htobe64(index->timestamp_begin); + + ret = ustctl_get_timestamp_end(ustream, &index->timestamp_end); + if (ret < 0) { + PERROR("ustctl_get_timestamp_end"); + goto error; + } + index->timestamp_end = htobe64(index->timestamp_end); + + ret = ustctl_get_events_discarded(ustream, &index->events_discarded); + if (ret < 0) { + PERROR("ustctl_get_events_discarded"); + goto error; + } + index->events_discarded = htobe64(index->events_discarded); + + ret = ustctl_get_content_size(ustream, &index->content_size); + if (ret < 0) { + PERROR("ustctl_get_content_size"); + goto error; + } + index->content_size = htobe64(index->content_size); + + ret = ustctl_get_packet_size(ustream, &index->packet_size); + if (ret < 0) { + PERROR("ustctl_get_packet_size"); + goto error; + } + index->packet_size = htobe64(index->packet_size); + + ret = ustctl_get_stream_id(ustream, &index->stream_id); + if (ret < 0) { + PERROR("ustctl_get_stream_id"); + goto error; + } + index->stream_id = htobe64(index->stream_id); + +error: + return ret; +} + + int lttng_ustconsumer_read_subbuffer(struct lttng_consumer_stream *stream, struct lttng_consumer_local_data *ctx) { unsigned long len, subbuf_size, padding; - int err; + int err, write_index = 0; long ret = 0; char dummy; struct ustctl_consumer_stream *ustream; + struct lttng_packet_index index; assert(stream); assert(stream->ustream); @@ -1619,6 +1678,11 @@ int lttng_ustconsumer_read_subbuffer(struct lttng_consumer_stream *stream, /* Ease our life for what's next. */ ustream = stream->ustream; + /* Indicate that for this stream we have to write the index. */ + if (stream->index_fd >= 0) { + write_index = 1; + } + /* We can consume the 1 byte written into the wait_fd by UST */ if (stream->monitor && !stream->hangup_flush_done) { ssize_t readlen; @@ -1676,6 +1740,15 @@ retry: goto end; } assert(stream->chan->output == CONSUMER_CHANNEL_MMAP); + + if (!stream->metadata_flag && write_index) { + index.offset = htobe64(stream->out_fd_offset); + ret = get_index_values(&index, ustream); + if (ret < 0) { + goto end; + } + } + /* Get the full padded subbuffer size */ err = ustctl_get_padded_subbuf_size(ustream, &len); assert(err == 0); @@ -1689,7 +1762,7 @@ retry: padding = len - subbuf_size; /* write the subbuffer to the tracefile */ - ret = lttng_consumer_on_read_subbuffer_mmap(ctx, stream, subbuf_size, padding); + ret = lttng_consumer_on_read_subbuffer_mmap(ctx, stream, subbuf_size, padding, &index); /* * The mmap operation should write subbuf_size amount of data when network * streaming or the full padding (len) size when we are _not_ streaming. @@ -1707,10 +1780,20 @@ retry: DBG("Error writing to tracefile " "(ret: %ld != len: %lu != subbuf_size: %lu)", ret, len, subbuf_size); + write_index = 0; } err = ustctl_put_next_subbuf(ustream); assert(err == 0); + /* Write index if needed. */ + if (write_index) { + err = index_write(stream->index_fd, &index, sizeof(index)); + if (err < 0) { + ret = -1; + goto end; + } + } + end: return ret; } @@ -1730,12 +1813,23 @@ int lttng_ustconsumer_on_recv_stream(struct lttng_consumer_stream *stream) if (stream->net_seq_idx == (uint64_t) -1ULL && stream->chan->monitor) { ret = utils_create_stream_file(stream->chan->pathname, stream->name, stream->chan->tracefile_size, stream->tracefile_count_current, - stream->uid, stream->gid); + stream->uid, stream->gid, NULL); if (ret < 0) { goto error; } stream->out_fd = ret; stream->tracefile_size_current = 0; + + if (!stream->metadata_flag) { + ret = index_create_file(stream->chan->pathname, + stream->name, stream->uid, stream->gid, + stream->chan->tracefile_size, + stream->tracefile_count_current); + if (ret < 0) { + goto error; + } + stream->index_fd = ret; + } } ret = 0; diff --git a/src/common/utils.c b/src/common/utils.c index b345100ea..dd9967312 100644 --- a/src/common/utils.c +++ b/src/common/utils.c @@ -351,10 +351,11 @@ error: */ LTTNG_HIDDEN int utils_create_stream_file(const char *path_name, char *file_name, uint64_t size, - uint64_t count, int uid, int gid) + uint64_t count, int uid, int gid, char *suffix) { int ret, out_fd, flags, mode; - char full_path[PATH_MAX], *path_name_id = NULL, *path; + char full_path[PATH_MAX], *path_name_suffix = NULL, *path; + char *extra = NULL; assert(path_name); assert(file_name); @@ -366,17 +367,30 @@ int utils_create_stream_file(const char *path_name, char *file_name, uint64_t si goto error; } + /* Setup extra string if suffix or/and a count is needed. */ + if (size > 0 && suffix) { + ret = asprintf(&extra, "_%" PRIu64 "%s", count, suffix); + } else if (size > 0) { + ret = asprintf(&extra, "_%" PRIu64, count); + } else if (suffix) { + ret = asprintf(&extra, "%s", suffix); + } + if (ret < 0) { + PERROR("Allocating extra string to name"); + goto error; + } + /* * If we split the trace in multiple files, we have to add the count at the * end of the tracefile name */ - if (size > 0) { - ret = asprintf(&path_name_id, "%s_%" PRIu64, full_path, count); + if (extra) { + ret = asprintf(&path_name_suffix, "%s%s", full_path, extra); if (ret < 0) { - PERROR("Allocating path name ID"); - goto error; + PERROR("Allocating path name with extra string"); + goto error_free_suffix; } - path = path_name_id; + path = path_name_suffix; } else { path = full_path; } @@ -397,7 +411,9 @@ int utils_create_stream_file(const char *path_name, char *file_name, uint64_t si ret = out_fd; error_open: - free(path_name_id); + free(path_name_suffix); +error_free_suffix: + free(extra); error: return ret; } @@ -413,10 +429,14 @@ error: */ LTTNG_HIDDEN int utils_rotate_stream_file(char *path_name, char *file_name, uint64_t size, - uint64_t count, int uid, int gid, int out_fd, uint64_t *new_count) + uint64_t count, int uid, int gid, int out_fd, uint64_t *new_count, + int *stream_fd) { int ret; + assert(new_count); + assert(stream_fd); + ret = close(out_fd); if (ret < 0) { PERROR("Closing tracefile"); @@ -429,8 +449,16 @@ int utils_rotate_stream_file(char *path_name, char *file_name, uint64_t size, (*new_count)++; } - return utils_create_stream_file(path_name, file_name, size, *new_count, - uid, gid); + ret = utils_create_stream_file(path_name, file_name, size, *new_count, + uid, gid, 0); + if (ret < 0) { + goto error; + } + *stream_fd = ret; + + /* Success. */ + ret = 0; + error: return ret; } diff --git a/src/common/utils.h b/src/common/utils.h index 7d8d70b5c..f7241996d 100644 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -36,9 +36,10 @@ int utils_set_fd_cloexec(int fd); int utils_create_pid_file(pid_t pid, const char *filepath); int utils_mkdir_recursive(const char *path, mode_t mode); int utils_create_stream_file(const char *path_name, char *file_name, uint64_t size, - uint64_t count, int uid, int gid); + uint64_t count, int uid, int gid, char *suffix); int utils_rotate_stream_file(char *path_name, char *file_name, uint64_t size, - uint64_t count, int uid, int gid, int out_fd, uint64_t *new_count); + uint64_t count, int uid, int gid, int out_fd, uint64_t *new_count, + int *stream_fd); int utils_parse_size_suffix(char *str, uint64_t *size); int utils_get_count_order_u32(uint32_t x); char *utils_get_home_dir(void); -- 2.34.1