[lttng-modules.git] / src / lttng-ring-buffer-event-notifier-client.h

/* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1)
 *
 * lttng-ring-buffer-event-notifier-client.h
 *
 * LTTng lib ring buffer event notifier client template.
 *
 * Copyright (C) 2010-2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 */

#include <linux/limits.h>
#include <linux/module.h>
#include <linux/types.h>
#include <wrapper/vmalloc.h>	/* for wrapper_vmalloc_sync_mappings() */
#include <lttng/abi.h>
#include <lttng/events.h>
#include <lttng/events-internal.h>
#include <lttng/tracer.h>
#include <wrapper/limits.h>

static struct lttng_transport lttng_relay_transport;

struct event_notifier_packet_header {
	uint8_t  header_end[0];
};

struct event_notifier_record_header {
	uint32_t payload_len;		/* in bytes */
	uint8_t header_end[0];		/* End of header */
};

static const struct lttng_kernel_ring_buffer_config client_config;

static inline
u64 lib_ring_buffer_clock_read(struct lttng_kernel_ring_buffer_channel *chan)
{
	return 0;
}

static inline
size_t record_header_size(const struct lttng_kernel_ring_buffer_config *config,
				 struct lttng_kernel_ring_buffer_channel *chan, size_t offset,
				 size_t *pre_header_padding,
				 struct lttng_kernel_ring_buffer_ctx *ctx,
				 void *client_ctx)
{
	size_t orig_offset = offset;
	size_t padding;

	padding = lib_ring_buffer_align(offset, lttng_alignof(uint32_t));
	offset += padding;

	offset += sizeof(uint32_t);

	*pre_header_padding = padding;

	return offset - orig_offset;
}

#include <ringbuffer/api.h>

static u64 client_ring_buffer_clock_read(struct lttng_kernel_ring_buffer_channel *chan)
{
	return 0;
}

static
size_t client_record_header_size(const struct lttng_kernel_ring_buffer_config *config,
				 struct lttng_kernel_ring_buffer_channel *chan, size_t offset,
				 size_t *pre_header_padding,
				 struct lttng_kernel_ring_buffer_ctx *ctx,
				 void *client_ctx)
{
	return record_header_size(config, chan, offset,
				  pre_header_padding, ctx, client_ctx);
}

/**
 * client_packet_header_size - called on buffer-switch to a new sub-buffer
 *
 * Return header size without padding after the structure. Don't use packed
 * structure because gcc generates inefficient code on some architectures
 * (powerpc, mips..)
 */
static size_t client_packet_header_size(void)
{
	return offsetof(struct event_notifier_packet_header, header_end);
}

static void client_buffer_begin(struct lttng_kernel_ring_buffer *buf, u64 tsc,
				unsigned int subbuf_idx)
{
}

/*
 * offset is assumed to never be 0 here : never deliver a completely empty
 * subbuffer. data_size is between 1 and subbuf_size.
 */
static void client_buffer_end(struct lttng_kernel_ring_buffer *buf, u64 tsc,
			      unsigned int subbuf_idx, unsigned long data_size)
{
}

static int client_buffer_create(struct lttng_kernel_ring_buffer *buf, void *priv,
				int cpu, const char *name)
{
	return 0;
}

static void client_buffer_finalize(struct lttng_kernel_ring_buffer *buf, void *priv, int cpu)
{
}

static int client_timestamp_begin(const struct lttng_kernel_ring_buffer_config *config,
		struct lttng_kernel_ring_buffer *buf, uint64_t *timestamp_begin)
{
	return -ENOSYS;
}

static int client_timestamp_end(const struct lttng_kernel_ring_buffer_config *config,
			struct lttng_kernel_ring_buffer *bufb,
			uint64_t *timestamp_end)
{
	return -ENOSYS;
}

static int client_events_discarded(const struct lttng_kernel_ring_buffer_config *config,
			struct lttng_kernel_ring_buffer *bufb,
			uint64_t *events_discarded)
{
	return -ENOSYS;
}

static int client_current_timestamp(const struct lttng_kernel_ring_buffer_config *config,
		struct lttng_kernel_ring_buffer *bufb,
		uint64_t *ts)
{
	return -ENOSYS;
}

static int client_content_size(const struct lttng_kernel_ring_buffer_config *config,
			struct lttng_kernel_ring_buffer *bufb,
			uint64_t *content_size)
{
	return -ENOSYS;
}

static int client_packet_size(const struct lttng_kernel_ring_buffer_config *config,
			struct lttng_kernel_ring_buffer *bufb,
			uint64_t *packet_size)
{
	return -ENOSYS;
}

static int client_stream_id(const struct lttng_kernel_ring_buffer_config *config,
			struct lttng_kernel_ring_buffer *bufb,
			uint64_t *stream_id)
{
	return -ENOSYS;
}

static int client_sequence_number(const struct lttng_kernel_ring_buffer_config *config,
			struct lttng_kernel_ring_buffer *bufb,
			uint64_t *seq)
{
	return -ENOSYS;
}

static
int client_instance_id(const struct lttng_kernel_ring_buffer_config *config,
		struct lttng_kernel_ring_buffer *bufb,
		uint64_t *id)
{
	return -ENOSYS;
}

static void client_record_get(const struct lttng_kernel_ring_buffer_config *config,
	struct lttng_kernel_ring_buffer_channel *chan, struct lttng_kernel_ring_buffer *buf,
	size_t offset, size_t *header_len,
	size_t *payload_len, u64 *timestamp)
{
	struct event_notifier_record_header header;
	int ret;

	ret = lib_ring_buffer_read(&buf->backend, offset, &header,
			offsetof(struct event_notifier_record_header, header_end));
	CHAN_WARN_ON(chan, ret != offsetof(struct event_notifier_record_header, header_end));
	*header_len = offsetof(struct event_notifier_record_header, header_end);
	*payload_len = header.payload_len;
	*timestamp = 0;
}

static const struct lttng_kernel_ring_buffer_config client_config = {
	.cb.ring_buffer_clock_read = client_ring_buffer_clock_read,
	.cb.record_header_size = client_record_header_size,
	.cb.subbuffer_header_size = client_packet_header_size,
	.cb.buffer_begin = client_buffer_begin,
	.cb.buffer_end = client_buffer_end,
	.cb.buffer_create = client_buffer_create,
	.cb.buffer_finalize = client_buffer_finalize,
	.cb.record_get = client_record_get,

	.tsc_bits = 0,
	.alloc = RING_BUFFER_ALLOC_GLOBAL,
	.sync = RING_BUFFER_SYNC_GLOBAL,
	.mode = RING_BUFFER_MODE_TEMPLATE,
	.backend = RING_BUFFER_PAGE,
	.output = RING_BUFFER_OUTPUT_TEMPLATE,
	.oops = RING_BUFFER_OOPS_CONSISTENCY,
	.ipi = RING_BUFFER_NO_IPI_BARRIER,
	.wakeup = RING_BUFFER_WAKEUP_BY_WRITER,
};

static
void release_priv_ops(void *priv_ops)
{
	module_put(THIS_MODULE);
}

static
void lttng_channel_destroy(struct lttng_kernel_ring_buffer_channel *chan)
{
	channel_destroy(chan);
}

static
struct lttng_kernel_ring_buffer_channel *_channel_create(const char *name,
				void *priv, void *buf_addr,
				size_t subbuf_size, size_t num_subbuf,
				unsigned int switch_timer_interval,
				unsigned int read_timer_interval)
{
	struct lttng_event_notifier_group *event_notifier_group = priv;
	struct lttng_kernel_ring_buffer_channel *chan;

	chan = channel_create(&client_config, name,
			      event_notifier_group, buf_addr,
			      subbuf_size, num_subbuf, switch_timer_interval,
			      read_timer_interval);
	if (chan) {
		/*
		 * Ensure this module is not unloaded before we finish
		 * using lttng_relay_transport.ops.
		 */
		if (!try_module_get(THIS_MODULE)) {
			printk(KERN_WARNING "LTTng: Can't lock event_notifier transport module.\n");
			goto error;
		}
		chan->backend.priv_ops = &lttng_relay_transport.ops;
		chan->backend.release_priv_ops = release_priv_ops;
	}
	return chan;

error:
	lttng_channel_destroy(chan);
	return NULL;
}

static
struct lttng_kernel_ring_buffer *lttng_buffer_read_open(struct lttng_kernel_ring_buffer_channel *chan)
{
	struct lttng_kernel_ring_buffer *buf;

	buf = channel_get_ring_buffer(&client_config, chan, 0);
	if (!lib_ring_buffer_open_read(buf))
		return buf;
	return NULL;
}

static
int lttng_buffer_has_read_closed_stream(struct lttng_kernel_ring_buffer_channel *chan)
{
	struct lttng_kernel_ring_buffer *buf;
	int cpu;

	for_each_channel_cpu(cpu, chan) {
		buf = channel_get_ring_buffer(&client_config, chan, cpu);
		if (!atomic_long_read(&buf->active_readers))
			return 1;
	}
	return 0;
}

static
void lttng_buffer_read_close(struct lttng_kernel_ring_buffer *buf)
{
	lib_ring_buffer_release_read(buf);
}

static
void lttng_write_event_notifier_header(const struct lttng_kernel_ring_buffer_config *config,
			    struct lttng_kernel_ring_buffer_ctx *ctx)
{
	uint32_t data_size;

	WARN_ON_ONCE(ctx->data_size > U32_MAX);

	data_size = (uint32_t) ctx->data_size;

	lib_ring_buffer_write(config, ctx, &data_size, sizeof(data_size));

	lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
}

static
int lttng_event_reserve(struct lttng_kernel_ring_buffer_ctx *ctx)
{
	struct lttng_kernel_ring_buffer_channel *chan = ctx->client_priv;
	int ret;

	memset(&ctx->priv, 0, sizeof(ctx->priv));
	ctx->priv.chan = chan;

	ret = lib_ring_buffer_reserve(&client_config, ctx, NULL);
	if (ret)
		return ret;
	lib_ring_buffer_backend_get_pages(&client_config, ctx,
			&ctx->priv.backend_pages);

	lttng_write_event_notifier_header(&client_config, ctx);
	return 0;
}

static
void lttng_event_commit(struct lttng_kernel_ring_buffer_ctx *ctx)
{
	lib_ring_buffer_commit(&client_config, ctx);
}

static
void lttng_event_write(struct lttng_kernel_ring_buffer_ctx *ctx, const void *src,
		     size_t len, size_t alignment)
{
	lib_ring_buffer_align_ctx(ctx, alignment);
	lib_ring_buffer_write(&client_config, ctx, src, len);
}

static
void lttng_event_write_from_user(struct lttng_kernel_ring_buffer_ctx *ctx,
			       const void __user *src, size_t len, size_t alignment)
{
	lib_ring_buffer_align_ctx(ctx, alignment);
	lib_ring_buffer_copy_from_user_inatomic(&client_config, ctx, src, len);
}

static
void lttng_event_memset(struct lttng_kernel_ring_buffer_ctx *ctx,
		int c, size_t len)
{
	lib_ring_buffer_memset(&client_config, ctx, c, len);
}

static
void lttng_event_strcpy(struct lttng_kernel_ring_buffer_ctx *ctx, const char *src,
		size_t len)
{
	lib_ring_buffer_strcpy(&client_config, ctx, src, len, '#');
}

static
size_t lttng_packet_avail_size(struct lttng_kernel_ring_buffer_channel *chan)
{
	unsigned long o_begin;
	struct lttng_kernel_ring_buffer *buf;

	buf = chan->backend.buf;	/* Only for global buffer ! */
	o_begin = v_read(&client_config, &buf->offset);
	if (subbuf_offset(o_begin, chan) != 0) {
		return chan->backend.subbuf_size - subbuf_offset(o_begin, chan);
	} else {
		return chan->backend.subbuf_size - subbuf_offset(o_begin, chan)
			- sizeof(struct event_notifier_packet_header);
	}
}

static
wait_queue_head_t *lttng_get_writer_buf_wait_queue(struct lttng_kernel_ring_buffer_channel *chan, int cpu)
{
	struct lttng_kernel_ring_buffer *buf = channel_get_ring_buffer(&client_config,
					chan, cpu);
	return &buf->write_wait;
}

static
wait_queue_head_t *lttng_get_hp_wait_queue(struct lttng_kernel_ring_buffer_channel *chan)
{
	return &chan->hp_wait;
}

static
int lttng_is_finalized(struct lttng_kernel_ring_buffer_channel *chan)
{
	return lib_ring_buffer_channel_is_finalized(chan);
}

static
int lttng_is_disabled(struct lttng_kernel_ring_buffer_channel *chan)
{
	return lib_ring_buffer_channel_is_disabled(chan);
}

static struct lttng_transport lttng_relay_transport = {
	.name = "relay-" RING_BUFFER_MODE_TEMPLATE_STRING,
	.owner = THIS_MODULE,
	.ops = {
		.priv = __LTTNG_COMPOUND_LITERAL(struct lttng_kernel_channel_buffer_ops_private, {
			.pub = &lttng_relay_transport.ops,
			.channel_create = _channel_create,
			.channel_destroy = lttng_channel_destroy,
			.buffer_read_open = lttng_buffer_read_open,
			.buffer_has_read_closed_stream =
				lttng_buffer_has_read_closed_stream,
			.buffer_read_close = lttng_buffer_read_close,
			.packet_avail_size = lttng_packet_avail_size,
			.get_writer_buf_wait_queue = lttng_get_writer_buf_wait_queue,
			.get_hp_wait_queue = lttng_get_hp_wait_queue,
			.is_finalized = lttng_is_finalized,
			.is_disabled = lttng_is_disabled,
			.timestamp_begin = client_timestamp_begin,
			.timestamp_end = client_timestamp_end,
			.events_discarded = client_events_discarded,
			.content_size = client_content_size,
			.packet_size = client_packet_size,
			.stream_id = client_stream_id,
			.current_timestamp = client_current_timestamp,
			.sequence_number = client_sequence_number,
			.instance_id = client_instance_id,
		}),
		.event_reserve = lttng_event_reserve,
		.event_commit = lttng_event_commit,
		.event_write_from_user = lttng_event_write_from_user,
		.event_memset = lttng_event_memset,
		.event_write = lttng_event_write,
		.event_strcpy = lttng_event_strcpy,
	},
};

static int __init lttng_ring_buffer_event_notifier_client_init(void)
{
	/*
	 * This vmalloc sync all also takes care of the lib ring buffer
	 * vmalloc'd module pages when it is built as a module into LTTng.
	 */
	wrapper_vmalloc_sync_mappings();
	lttng_transport_register(&lttng_relay_transport);
	return 0;
}

module_init(lttng_ring_buffer_event_notifier_client_init);

static void __exit lttng_ring_buffer_event_notifier_client_exit(void)
{
	lttng_transport_unregister(&lttng_relay_transport);
}

module_exit(lttng_ring_buffer_event_notifier_client_exit);

MODULE_LICENSE("GPL and additional rights");
MODULE_AUTHOR("Mathieu Desnoyers <mathieu.desnoyers@efficios.com>");
MODULE_DESCRIPTION("LTTng ring buffer " RING_BUFFER_MODE_TEMPLATE_STRING
		   " client");
MODULE_VERSION(__stringify(LTTNG_MODULES_MAJOR_VERSION) "."
	__stringify(LTTNG_MODULES_MINOR_VERSION) "."
	__stringify(LTTNG_MODULES_PATCHLEVEL_VERSION)
	LTTNG_MODULES_EXTRAVERSION);
Commit	Line	Data
250c663f MD	1	/* SPDX-License-Identifier: (GPL-2.0 or LGPL-2.1)
	2	*
	3	* lttng-ring-buffer-event-notifier-client.h
	4	*
	5	* LTTng lib ring buffer event notifier client template.
	6	*
	7	* Copyright (C) 2010-2020 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	8	*/
	9
99d223ad	10	#include <linux/limits.h>
250c663f MD	11	#include <linux/module.h>
	12	#include <linux/types.h>
	13	#include <wrapper/vmalloc.h> /* for wrapper_vmalloc_sync_mappings() */
99d223ad	14	#include <lttng/abi.h>
250c663f	15	#include <lttng/events.h>
e8c0bbd3	16	#include <lttng/events-internal.h>
250c663f	17	#include <lttng/tracer.h>
236233f7	18	#include <wrapper/limits.h>
250c663f MD	19
	20	static struct lttng_transport lttng_relay_transport;
	21
	22	struct event_notifier_packet_header {
	23	uint8_t header_end[0];
	24	};
	25
	26	struct event_notifier_record_header {
99d223ad	27	uint32_t payload_len; /* in bytes */
250c663f MD	28	uint8_t header_end[0]; /* End of header */
	29	};
	30
e20c0fec	31	static const struct lttng_kernel_ring_buffer_config client_config;
250c663f MD	32
250c663f MD	33	static inline
860c213b	34	u64 lib_ring_buffer_clock_read(struct lttng_kernel_ring_buffer_channel *chan)
250c663f MD	35	{
	36	return 0;
	37	}
	38
	39	static inline
e20c0fec	40	size_t record_header_size(const struct lttng_kernel_ring_buffer_config *config,
860c213b	41	struct lttng_kernel_ring_buffer_channel *chan, size_t offset,
250c663f	42	size_t *pre_header_padding,
8a57ec02	43	struct lttng_kernel_ring_buffer_ctx *ctx,
250c663f MD	44	void *client_ctx)
250c663f MD	45	{
99d223ad FD	46	size_t orig_offset = offset;
	47	size_t padding;
	48
	49	padding = lib_ring_buffer_align(offset, lttng_alignof(uint32_t));
	50	offset += padding;
	51
	52	offset += sizeof(uint32_t);
	53
	54	*pre_header_padding = padding;
	55
	56	return offset - orig_offset;
250c663f MD	57	}
	58
	59	#include <ringbuffer/api.h>
	60
860c213b	61	static u64 client_ring_buffer_clock_read(struct lttng_kernel_ring_buffer_channel *chan)
250c663f MD	62	{
	63	return 0;
	64	}
	65
	66	static
e20c0fec	67	size_t client_record_header_size(const struct lttng_kernel_ring_buffer_config *config,
860c213b	68	struct lttng_kernel_ring_buffer_channel *chan, size_t offset,
250c663f	69	size_t *pre_header_padding,
8a57ec02	70	struct lttng_kernel_ring_buffer_ctx *ctx,
250c663f MD	71	void *client_ctx)
250c663f MD	72	{
99d223ad FD	73	return record_header_size(config, chan, offset,
99d223ad FD	74	pre_header_padding, ctx, client_ctx);
250c663f MD	75	}
	76
	77	/**
	78	* client_packet_header_size - called on buffer-switch to a new sub-buffer
	79	*
	80	* Return header size without padding after the structure. Don't use packed
	81	* structure because gcc generates inefficient code on some architectures
	82	* (powerpc, mips..)
	83	*/
	84	static size_t client_packet_header_size(void)
	85	{
	86	return offsetof(struct event_notifier_packet_header, header_end);
	87	}
	88
e20c0fec	89	static void client_buffer_begin(struct lttng_kernel_ring_buffer *buf, u64 tsc,
250c663f MD	90	unsigned int subbuf_idx)
	91	{
	92	}
	93
	94	/*
	95	* offset is assumed to never be 0 here : never deliver a completely empty
	96	* subbuffer. data_size is between 1 and subbuf_size.
	97	*/
e20c0fec	98	static void client_buffer_end(struct lttng_kernel_ring_buffer *buf, u64 tsc,
250c663f MD	99	unsigned int subbuf_idx, unsigned long data_size)
	100	{
	101	}
	102
e20c0fec	103	static int client_buffer_create(struct lttng_kernel_ring_buffer buf, void priv,
250c663f MD	104	int cpu, const char *name)
	105	{
	106	return 0;
	107	}
	108
e20c0fec	109	static void client_buffer_finalize(struct lttng_kernel_ring_buffer buf, void priv, int cpu)
250c663f MD	110	{
	111	}
	112
e20c0fec MD	113	static int client_timestamp_begin(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	114	struct lttng_kernel_ring_buffer buf, uint64_t timestamp_begin)
250c663f MD	115	{
	116	return -ENOSYS;
	117	}
	118
e20c0fec MD	119	static int client_timestamp_end(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	120	struct lttng_kernel_ring_buffer *bufb,
250c663f MD	121	uint64_t *timestamp_end)
	122	{
	123	return -ENOSYS;
	124	}
	125
e20c0fec MD	126	static int client_events_discarded(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	127	struct lttng_kernel_ring_buffer *bufb,
250c663f MD	128	uint64_t *events_discarded)
	129	{
	130	return -ENOSYS;
	131	}
	132
e20c0fec MD	133	static int client_current_timestamp(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	134	struct lttng_kernel_ring_buffer *bufb,
250c663f MD	135	uint64_t *ts)
	136	{
	137	return -ENOSYS;
	138	}
	139
e20c0fec MD	140	static int client_content_size(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	141	struct lttng_kernel_ring_buffer *bufb,
250c663f MD	142	uint64_t *content_size)
	143	{
	144	return -ENOSYS;
	145	}
	146
e20c0fec MD	147	static int client_packet_size(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	148	struct lttng_kernel_ring_buffer *bufb,
250c663f MD	149	uint64_t *packet_size)
	150	{
	151	return -ENOSYS;
	152	}
	153
e20c0fec MD	154	static int client_stream_id(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	155	struct lttng_kernel_ring_buffer *bufb,
250c663f MD	156	uint64_t *stream_id)
	157	{
	158	return -ENOSYS;
	159	}
	160
e20c0fec MD	161	static int client_sequence_number(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	162	struct lttng_kernel_ring_buffer *bufb,
250c663f MD	163	uint64_t *seq)
	164	{
	165	return -ENOSYS;
	166	}
	167
	168	static
e20c0fec MD	169	int client_instance_id(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	170	struct lttng_kernel_ring_buffer *bufb,
250c663f MD	171	uint64_t *id)
	172	{
	173	return -ENOSYS;
	174	}
	175
e20c0fec MD	176	static void client_record_get(const struct lttng_kernel_ring_buffer_config *config,
e20c0fec MD	177	struct lttng_kernel_ring_buffer_channel chan, struct lttng_kernel_ring_buffer buf,
250c663f MD	178	size_t offset, size_t *header_len,
	179	size_t payload_len, u64 timestamp)
	180	{
	181	struct event_notifier_record_header header;
	182	int ret;
	183
	184	ret = lib_ring_buffer_read(&buf->backend, offset, &header,
	185	offsetof(struct event_notifier_record_header, header_end));
	186	CHAN_WARN_ON(chan, ret != offsetof(struct event_notifier_record_header, header_end));
	187	*header_len = offsetof(struct event_notifier_record_header, header_end);
99d223ad	188	*payload_len = header.payload_len;
250c663f MD	189	*timestamp = 0;
	190	}
	191
e20c0fec	192	static const struct lttng_kernel_ring_buffer_config client_config = {
250c663f MD	193	.cb.ring_buffer_clock_read = client_ring_buffer_clock_read,
	194	.cb.record_header_size = client_record_header_size,
	195	.cb.subbuffer_header_size = client_packet_header_size,
	196	.cb.buffer_begin = client_buffer_begin,
	197	.cb.buffer_end = client_buffer_end,
	198	.cb.buffer_create = client_buffer_create,
	199	.cb.buffer_finalize = client_buffer_finalize,
	200	.cb.record_get = client_record_get,
	201
	202	.tsc_bits = 0,
	203	.alloc = RING_BUFFER_ALLOC_GLOBAL,
	204	.sync = RING_BUFFER_SYNC_GLOBAL,
	205	.mode = RING_BUFFER_MODE_TEMPLATE,
	206	.backend = RING_BUFFER_PAGE,
	207	.output = RING_BUFFER_OUTPUT_TEMPLATE,
	208	.oops = RING_BUFFER_OOPS_CONSISTENCY,
	209	.ipi = RING_BUFFER_NO_IPI_BARRIER,
	210	.wakeup = RING_BUFFER_WAKEUP_BY_WRITER,
	211	};
	212
	213	static
	214	void release_priv_ops(void *priv_ops)
	215	{
	216	module_put(THIS_MODULE);
	217	}
	218
	219	static
860c213b	220	void lttng_channel_destroy(struct lttng_kernel_ring_buffer_channel *chan)
250c663f MD	221	{
	222	channel_destroy(chan);
	223	}
	224
	225	static
860c213b	226	struct lttng_kernel_ring_buffer_channel _channel_create(const char name,
250c663f MD	227	void priv, void buf_addr,
	228	size_t subbuf_size, size_t num_subbuf,
	229	unsigned int switch_timer_interval,
	230	unsigned int read_timer_interval)
	231	{
	232	struct lttng_event_notifier_group *event_notifier_group = priv;
860c213b	233	struct lttng_kernel_ring_buffer_channel *chan;
250c663f MD	234
	235	chan = channel_create(&client_config, name,
	236	event_notifier_group, buf_addr,
	237	subbuf_size, num_subbuf, switch_timer_interval,
	238	read_timer_interval);
	239	if (chan) {
	240	/*
	241	* Ensure this module is not unloaded before we finish
	242	* using lttng_relay_transport.ops.
	243	*/
	244	if (!try_module_get(THIS_MODULE)) {
	245	printk(KERN_WARNING "LTTng: Can't lock event_notifier transport module.\n");
	246	goto error;
	247	}
	248	chan->backend.priv_ops = &lttng_relay_transport.ops;
	249	chan->backend.release_priv_ops = release_priv_ops;
	250	}
	251	return chan;
	252
	253	error:
	254	lttng_channel_destroy(chan);
	255	return NULL;
	256	}
	257
	258	static
e20c0fec	259	struct lttng_kernel_ring_buffer lttng_buffer_read_open(struct lttng_kernel_ring_buffer_channel chan)
250c663f	260	{
e20c0fec	261	struct lttng_kernel_ring_buffer *buf;
250c663f MD	262
	263	buf = channel_get_ring_buffer(&client_config, chan, 0);
	264	if (!lib_ring_buffer_open_read(buf))
	265	return buf;
	266	return NULL;
	267	}
	268
	269	static
860c213b	270	int lttng_buffer_has_read_closed_stream(struct lttng_kernel_ring_buffer_channel *chan)
250c663f	271	{
e20c0fec	272	struct lttng_kernel_ring_buffer *buf;
250c663f MD	273	int cpu;
	274
	275	for_each_channel_cpu(cpu, chan) {
	276	buf = channel_get_ring_buffer(&client_config, chan, cpu);
	277	if (!atomic_long_read(&buf->active_readers))
	278	return 1;
	279	}
	280	return 0;
	281	}
	282
	283	static
e20c0fec	284	void lttng_buffer_read_close(struct lttng_kernel_ring_buffer *buf)
250c663f MD	285	{
	286	lib_ring_buffer_release_read(buf);
	287	}
	288
99d223ad	289	static
e20c0fec	290	void lttng_write_event_notifier_header(const struct lttng_kernel_ring_buffer_config *config,
8a57ec02	291	struct lttng_kernel_ring_buffer_ctx *ctx)
99d223ad FD	292	{
	293	uint32_t data_size;
	294
	295	WARN_ON_ONCE(ctx->data_size > U32_MAX);
	296
	297	data_size = (uint32_t) ctx->data_size;
	298
	299	lib_ring_buffer_write(config, ctx, &data_size, sizeof(data_size));
	300
	301	lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
	302	}
	303
250c663f	304	static
8a57ec02	305	int lttng_event_reserve(struct lttng_kernel_ring_buffer_ctx *ctx)
250c663f	306	{
860c213b	307	struct lttng_kernel_ring_buffer_channel *chan = ctx->client_priv;
250c663f MD	308	int ret;
250c663f MD	309
b1199bd3 MD	310	memset(&ctx->priv, 0, sizeof(ctx->priv));
	311	ctx->priv.chan = chan;
	312
250c663f MD	313	ret = lib_ring_buffer_reserve(&client_config, ctx, NULL);
	314	if (ret)
	315	return ret;
	316	lib_ring_buffer_backend_get_pages(&client_config, ctx,
b1199bd3	317	&ctx->priv.backend_pages);
99d223ad FD	318
99d223ad FD	319	lttng_write_event_notifier_header(&client_config, ctx);
250c663f MD	320	return 0;
	321	}
	322
	323	static
8a57ec02	324	void lttng_event_commit(struct lttng_kernel_ring_buffer_ctx *ctx)
250c663f MD	325	{
	326	lib_ring_buffer_commit(&client_config, ctx);
	327	}
	328
	329	static
8a57ec02	330	void lttng_event_write(struct lttng_kernel_ring_buffer_ctx ctx, const void src,
f5ffbd77	331	size_t len, size_t alignment)
250c663f	332	{
f5ffbd77	333	lib_ring_buffer_align_ctx(ctx, alignment);
250c663f MD	334	lib_ring_buffer_write(&client_config, ctx, src, len);
	335	}
	336
	337	static
8a57ec02	338	void lttng_event_write_from_user(struct lttng_kernel_ring_buffer_ctx *ctx,
f5ffbd77	339	const void __user *src, size_t len, size_t alignment)
250c663f	340	{
f5ffbd77	341	lib_ring_buffer_align_ctx(ctx, alignment);
250c663f MD	342	lib_ring_buffer_copy_from_user_inatomic(&client_config, ctx, src, len);
	343	}
	344
	345	static
8a57ec02	346	void lttng_event_memset(struct lttng_kernel_ring_buffer_ctx *ctx,
250c663f MD	347	int c, size_t len)
	348	{
	349	lib_ring_buffer_memset(&client_config, ctx, c, len);
	350	}
	351
	352	static
8a57ec02	353	void lttng_event_strcpy(struct lttng_kernel_ring_buffer_ctx ctx, const char src,
250c663f MD	354	size_t len)
	355	{
	356	lib_ring_buffer_strcpy(&client_config, ctx, src, len, '#');
	357	}
	358
	359	static
860c213b	360	size_t lttng_packet_avail_size(struct lttng_kernel_ring_buffer_channel *chan)
250c663f MD	361	{
250c663f MD	362	unsigned long o_begin;
e20c0fec	363	struct lttng_kernel_ring_buffer *buf;
250c663f MD	364
	365	buf = chan->backend.buf; /* Only for global buffer ! */
	366	o_begin = v_read(&client_config, &buf->offset);
	367	if (subbuf_offset(o_begin, chan) != 0) {
	368	return chan->backend.subbuf_size - subbuf_offset(o_begin, chan);
	369	} else {
	370	return chan->backend.subbuf_size - subbuf_offset(o_begin, chan)
	371	- sizeof(struct event_notifier_packet_header);
	372	}
	373	}
	374
	375	static
860c213b	376	wait_queue_head_t lttng_get_writer_buf_wait_queue(struct lttng_kernel_ring_buffer_channel chan, int cpu)
250c663f	377	{
e20c0fec	378	struct lttng_kernel_ring_buffer *buf = channel_get_ring_buffer(&client_config,
250c663f MD	379	chan, cpu);
	380	return &buf->write_wait;
	381	}
	382
	383	static
860c213b	384	wait_queue_head_t lttng_get_hp_wait_queue(struct lttng_kernel_ring_buffer_channel chan)
250c663f MD	385	{
	386	return &chan->hp_wait;
	387	}
	388
	389	static
860c213b	390	int lttng_is_finalized(struct lttng_kernel_ring_buffer_channel *chan)
250c663f MD	391	{
	392	return lib_ring_buffer_channel_is_finalized(chan);
	393	}
	394
	395	static
860c213b	396	int lttng_is_disabled(struct lttng_kernel_ring_buffer_channel *chan)
250c663f MD	397	{
	398	return lib_ring_buffer_channel_is_disabled(chan);
	399	}
	400
	401	static struct lttng_transport lttng_relay_transport = {
	402	.name = "relay-" RING_BUFFER_MODE_TEMPLATE_STRING,
	403	.owner = THIS_MODULE,
	404	.ops = {
4a399b76 MD	405	.priv = __LTTNG_COMPOUND_LITERAL(struct lttng_kernel_channel_buffer_ops_private, {
	406	.pub = &lttng_relay_transport.ops,
	407	.channel_create = _channel_create,
	408	.channel_destroy = lttng_channel_destroy,
	409	.buffer_read_open = lttng_buffer_read_open,
	410	.buffer_has_read_closed_stream =
	411	lttng_buffer_has_read_closed_stream,
	412	.buffer_read_close = lttng_buffer_read_close,
	413	.packet_avail_size = lttng_packet_avail_size,
	414	.get_writer_buf_wait_queue = lttng_get_writer_buf_wait_queue,
	415	.get_hp_wait_queue = lttng_get_hp_wait_queue,
	416	.is_finalized = lttng_is_finalized,
	417	.is_disabled = lttng_is_disabled,
	418	.timestamp_begin = client_timestamp_begin,
	419	.timestamp_end = client_timestamp_end,
	420	.events_discarded = client_events_discarded,
	421	.content_size = client_content_size,
	422	.packet_size = client_packet_size,
	423	.stream_id = client_stream_id,
	424	.current_timestamp = client_current_timestamp,
	425	.sequence_number = client_sequence_number,
	426	.instance_id = client_instance_id,
	427	}),
250c663f MD	428	.event_reserve = lttng_event_reserve,
	429	.event_commit = lttng_event_commit,
	430	.event_write_from_user = lttng_event_write_from_user,
	431	.event_memset = lttng_event_memset,
	432	.event_write = lttng_event_write,
	433	.event_strcpy = lttng_event_strcpy,
250c663f MD	434	},
	435	};
	436
	437	static int __init lttng_ring_buffer_event_notifier_client_init(void)
	438	{
	439	/*
	440	* This vmalloc sync all also takes care of the lib ring buffer
	441	* vmalloc'd module pages when it is built as a module into LTTng.
	442	*/
	443	wrapper_vmalloc_sync_mappings();
	444	lttng_transport_register(&lttng_relay_transport);
	445	return 0;
	446	}
	447
	448	module_init(lttng_ring_buffer_event_notifier_client_init);
	449
	450	static void __exit lttng_ring_buffer_event_notifier_client_exit(void)
	451	{
	452	lttng_transport_unregister(&lttng_relay_transport);
	453	}
	454
	455	module_exit(lttng_ring_buffer_event_notifier_client_exit);
	456
	457	MODULE_LICENSE("GPL and additional rights");
	458	MODULE_AUTHOR("Mathieu Desnoyers <mathieu.desnoyers@efficios.com>");
	459	MODULE_DESCRIPTION("LTTng ring buffer " RING_BUFFER_MODE_TEMPLATE_STRING
	460	" client");
	461	MODULE_VERSION(__stringify(LTTNG_MODULES_MAJOR_VERSION) "."
	462	__stringify(LTTNG_MODULES_MINOR_VERSION) "."
	463	__stringify(LTTNG_MODULES_PATCHLEVEL_VERSION)
	464	LTTNG_MODULES_EXTRAVERSION);