[lttng-modules.git] / src / lttng-event-notifier-notification.c

/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
 *
 * lttng-event-notifier-notification.c
 *
 * Copyright (C) 2020 Francis Deslauriers <francis.deslauriers@efficios.com>
 */

#include <linux/bug.h>

#include <lttng/lttng-bytecode.h>
#include <lttng/events.h>
#include <lttng/msgpack.h>
#include <lttng/event-notifier-notification.h>
#include <lttng/events-internal.h>
#include <wrapper/barrier.h>

/*
 * The capture buffer size needs to be below 1024 bytes to avoid the
 * frame to be larger than the 1024 limit enforced by the kernel. If we
 * ever need to increase it, we will need to use a memory allocation
 * scheme which allows allocating temporary memory chunks from the
 * instrumentation sites. This could be done by adapting lttng
 * tp-mempool to become nmi-safe and lock-free.
 */
#define CAPTURE_BUFFER_SIZE 512

struct lttng_event_notifier_notification {
	int notification_fd;
	uint64_t event_notifier_token;
	uint8_t capture_buf[CAPTURE_BUFFER_SIZE];
	struct lttng_msgpack_writer writer;
	bool has_captures;
};

static
int capture_enum(struct lttng_msgpack_writer *writer,
		struct lttng_interpreter_output *output)
{
	int ret;

	/*
	 * Enums are captured as a map containing 2 key-value pairs. Such as:
	 * - type: enum
	 *   value: 177
	 */
	ret = lttng_msgpack_begin_map(writer, 2);
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = lttng_msgpack_write_str(writer, "type");
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = lttng_msgpack_write_str(writer, "enum");
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = lttng_msgpack_write_str(writer, "value");
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	switch (output->type) {
	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
		ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
		ret = lttng_msgpack_write_signed_integer(writer, output->u.u);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	default:
		WARN_ON(1);
	}

	ret = lttng_msgpack_end_map(writer);
	if (ret)
		WARN_ON_ONCE(1);

end:
	return ret;
}

static
int64_t capture_sequence_element_signed(uint8_t *ptr,
		const struct lttng_kernel_type_integer *type)
{
	int64_t value = 0;
	unsigned int size = type->size;
	bool byte_order_reversed = type->reverse_byte_order;

	switch (size) {
	case 8:
		value = *ptr;
		break;
	case 16:
	{
		int16_t tmp;
		tmp = *(int16_t *) ptr;
		if (byte_order_reversed)
			__swab16s(&tmp);

		value = tmp;
		break;
	}
	case 32:
	{
		int32_t tmp;
		tmp = *(int32_t *) ptr;
		if (byte_order_reversed)
			__swab32s(&tmp);

		value = tmp;
		break;
	}
	case 64:
	{
		int64_t tmp;
		tmp = *(int64_t *) ptr;
		if (byte_order_reversed)
			__swab64s(&tmp);

		value = tmp;
		break;
	}
	default:
		WARN_ON(1);
	}

	return value;
}

static
uint64_t capture_sequence_element_unsigned(uint8_t *ptr,
		const struct lttng_kernel_type_integer *type)
{
	uint64_t value = 0;
	unsigned int size = type->size;
	bool byte_order_reversed = type->reverse_byte_order;

	switch (size) {
	case 8:
		value = *ptr;
		break;
	case 16:
	{
		uint16_t tmp;
		tmp = *(uint16_t *) ptr;
		if (byte_order_reversed)
			__swab16s(&tmp);

		value = tmp;
		break;
	}
	case 32:
	{
		uint32_t tmp;
		tmp = *(uint32_t *) ptr;
		if (byte_order_reversed)
			__swab32s(&tmp);

		value = tmp;
		break;
	}
	case 64:
	{
		uint64_t tmp;
		tmp = *(uint64_t *) ptr;
		if (byte_order_reversed)
			__swab64s(&tmp);

		value = tmp;
		break;
	}
	default:
		WARN_ON(1);
	}

	return value;
}

int capture_sequence(struct lttng_msgpack_writer *writer,
		struct lttng_interpreter_output *output)
{
	const struct lttng_kernel_type_integer *integer_type = NULL;
	const struct lttng_kernel_type_common *nested_type;
	uint8_t *ptr;
	bool signedness;
	int ret, i;

	ret = lttng_msgpack_begin_array(writer, output->u.sequence.nr_elem);
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ptr = (uint8_t *) output->u.sequence.ptr;
	nested_type = output->u.sequence.nested_type;
	switch (nested_type->type) {
	case lttng_kernel_type_integer:
		integer_type = lttng_kernel_get_type_integer(nested_type);
		break;
	case lttng_kernel_type_enum:
		/* Treat enumeration as an integer. */
		integer_type = lttng_kernel_get_type_integer(lttng_kernel_get_type_enum(nested_type)->container_type);
		break;
	default:
		/* Capture of array of non-integer are not supported. */
		WARN_ON(1);
	}
	signedness = integer_type->signedness;
	for (i = 0; i < output->u.sequence.nr_elem; i++) {
		if (signedness) {
			ret = lttng_msgpack_write_signed_integer(writer,
				capture_sequence_element_signed(ptr, integer_type));
			if (ret) {
				WARN_ON_ONCE(1);
				goto end;
			}
		} else {
			ret = lttng_msgpack_write_unsigned_integer(writer,
				capture_sequence_element_unsigned(ptr, integer_type));
			if (ret) {
				WARN_ON_ONCE(1);
				goto end;
			}
		}

		/*
		 * We assume that alignment is smaller or equal to the size.
		 * This currently holds true but if it changes in the future,
		 * we will want to change the pointer arithmetics below to
		 * take into account that the next element might be further
		 * away.
		 */
		WARN_ON(integer_type->alignment > integer_type->size);

		/* Size is in number of bits. */
		ptr += (integer_type->size / CHAR_BIT) ;
	}

	ret = lttng_msgpack_end_array(writer);
	if (ret)
		WARN_ON_ONCE(1);
end:
	return ret;
}

static
int notification_append_capture(
		struct lttng_event_notifier_notification *notif,
		struct lttng_interpreter_output *output)
{
	struct lttng_msgpack_writer *writer = &notif->writer;
	int ret = 0;

	switch (output->type) {
	case LTTNG_INTERPRETER_TYPE_S64:
		ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_U64:
		ret = lttng_msgpack_write_unsigned_integer(writer, output->u.u);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_STRING:
		if (output->u.str.user) {
			ret = lttng_msgpack_write_user_str(writer, output->u.str.user_str);
		} else {
			ret = lttng_msgpack_write_str(writer, output->u.str.str);
		}
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_SEQUENCE:
		ret = capture_sequence(writer, output);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
		ret = capture_enum(writer, output);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	default:
		ret = -1;
		WARN_ON(1);
	}
end:
	return ret;
}

static
int notification_append_empty_capture(
		struct lttng_event_notifier_notification *notif)
{
	int ret = lttng_msgpack_write_nil(&notif->writer);
	if (ret)
		WARN_ON_ONCE(1);

	return ret;
}

static
int notification_init(struct lttng_event_notifier_notification *notif,
		struct lttng_kernel_event_notifier *event_notifier)
{
	struct lttng_msgpack_writer *writer = &notif->writer;
	int ret = 0;

	notif->has_captures = false;

	if (event_notifier->priv->num_captures > 0) {
		lttng_msgpack_writer_init(writer, notif->capture_buf,
				CAPTURE_BUFFER_SIZE);

		ret = lttng_msgpack_begin_array(writer, event_notifier->priv->num_captures);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}

		notif->has_captures = true;
	}

end:
	return ret;
}

static
void record_error(struct lttng_kernel_event_notifier *event_notifier)
{

	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
	struct lttng_counter *error_counter;
	size_t dimension_index[1];
	int ret;

	/*
	 * lttng_smp_load_acquire paired with lttng_smp_store_release orders
	 * creation of the error counter and setting error_counter_len
	 * before the error_counter is used.
	 */
	error_counter = lttng_smp_load_acquire(&event_notifier_group->error_counter);
	/* This group may not have an error counter attached to it. */
	if (!error_counter)
		return;

	dimension_index[0] = event_notifier->priv->error_counter_index;

	ret = error_counter->ops->counter_add(error_counter->counter,
			dimension_index, 1);
	if (ret)
		WARN_ON_ONCE(1);
}

static
void notification_send(struct lttng_event_notifier_notification *notif,
		struct lttng_kernel_event_notifier *event_notifier)
{
	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
	struct lttng_kernel_ring_buffer_ctx ctx;
	struct lttng_kernel_abi_event_notifier_notification kernel_notif;
	size_t capture_buffer_content_len, reserve_size;
	int ret;

	reserve_size = sizeof(kernel_notif);
	kernel_notif.token = event_notifier->priv->parent.user_token;

	if (notif->has_captures) {
		capture_buffer_content_len = notif->writer.write_pos - notif->writer.buffer;
	} else {
		capture_buffer_content_len = 0;
	}

	WARN_ON_ONCE(capture_buffer_content_len > CAPTURE_BUFFER_SIZE);

	reserve_size += capture_buffer_content_len;
	kernel_notif.capture_buf_size = capture_buffer_content_len;

	lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, reserve_size,
			lttng_alignof(kernel_notif), NULL);
	ret = event_notifier_group->ops->event_reserve(&ctx);
	if (ret < 0) {
		record_error(event_notifier);
		return;
	}

	/* Write the notif structure. */
	event_notifier_group->ops->event_write(&ctx, &kernel_notif,
			sizeof(kernel_notif), lttng_alignof(kernel_notif));

	/*
	 * Write the capture buffer. No need to realigned as the below is a raw
	 * char* buffer.
	 */
	event_notifier_group->ops->event_write(&ctx, &notif->capture_buf,
			capture_buffer_content_len, 1);

	event_notifier_group->ops->event_commit(&ctx);
	irq_work_queue(&event_notifier_group->wakeup_pending);
}

void lttng_event_notifier_notification_send(struct lttng_kernel_event_notifier *event_notifier,
		const char *stack_data,
		struct lttng_kernel_probe_ctx *probe_ctx,
		struct lttng_kernel_notification_ctx *notif_ctx)
{
	struct lttng_event_notifier_notification notif = { 0 };
	int ret;

	if (unlikely(!READ_ONCE(event_notifier->parent.enabled)))
		return;

	ret = notification_init(&notif, event_notifier);
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	if (unlikely(notif_ctx->eval_capture)) {
		struct lttng_kernel_bytecode_runtime *capture_bc_runtime;

		/*
		 * Iterate over all the capture bytecodes. If the interpreter
		 * functions returns successfully, append the value of the
		 * `output` parameter to the capture buffer. If the interpreter
		 * fails, append an empty capture to the buffer.
		 */
		list_for_each_entry_rcu(capture_bc_runtime,
				&event_notifier->priv->capture_bytecode_runtime_head, node) {
			struct lttng_interpreter_output output;

			if (capture_bc_runtime->interpreter_func(capture_bc_runtime,
					stack_data, probe_ctx, &output) == LTTNG_KERNEL_BYTECODE_INTERPRETER_OK)
				ret = notification_append_capture(&notif, &output);
			else
				ret = notification_append_empty_capture(&notif);

			if (ret)
				printk(KERN_WARNING "Error appending capture to notification");
		}
	}

	/*
	 * Send the notification (including the capture buffer) to the
	 * sessiond.
	 */
	notification_send(&notif, event_notifier);
end:
	return;
}
Commit	Line	Data
21f58fb7 FD	1	/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
	2	*
	3	* lttng-event-notifier-notification.c
	4	*
	5	* Copyright (C) 2020 Francis Deslauriers <francis.deslauriers@efficios.com>
	6	*/
	7
42f3ef8c FD	8	#include <linux/bug.h>
	9
	10	#include <lttng/lttng-bytecode.h>
21f58fb7	11	#include <lttng/events.h>
42f3ef8c	12	#include <lttng/msgpack.h>
21f58fb7	13	#include <lttng/event-notifier-notification.h>
437d5aa5	14	#include <lttng/events-internal.h>
6657edec	15	#include <wrapper/barrier.h>
21f58fb7	16
99d223ad	17	/*
e699ee6c MD	18	* The capture buffer size needs to be below 1024 bytes to avoid the
	19	* frame to be larger than the 1024 limit enforced by the kernel. If we
	20	* ever need to increase it, we will need to use a memory allocation
	21	* scheme which allows allocating temporary memory chunks from the
	22	* instrumentation sites. This could be done by adapting lttng
	23	* tp-mempool to become nmi-safe and lock-free.
99d223ad FD	24	*/
	25	#define CAPTURE_BUFFER_SIZE 512
	26
	27	struct lttng_event_notifier_notification {
	28	int notification_fd;
	29	uint64_t event_notifier_token;
	30	uint8_t capture_buf[CAPTURE_BUFFER_SIZE];
	31	struct lttng_msgpack_writer writer;
	32	bool has_captures;
	33	};
	34
42f3ef8c FD	35	static
	36	int capture_enum(struct lttng_msgpack_writer *writer,
	37	struct lttng_interpreter_output *output)
	38	{
	39	int ret;
	40
	41	/*
	42	* Enums are captured as a map containing 2 key-value pairs. Such as:
	43	* - type: enum
	44	* value: 177
	45	*/
	46	ret = lttng_msgpack_begin_map(writer, 2);
	47	if (ret) {
	48	WARN_ON_ONCE(1);
	49	goto end;
	50	}
	51
	52	ret = lttng_msgpack_write_str(writer, "type");
	53	if (ret) {
	54	WARN_ON_ONCE(1);
	55	goto end;
	56	}
	57
	58	ret = lttng_msgpack_write_str(writer, "enum");
	59	if (ret) {
	60	WARN_ON_ONCE(1);
	61	goto end;
	62	}
	63
	64	ret = lttng_msgpack_write_str(writer, "value");
	65	if (ret) {
	66	WARN_ON_ONCE(1);
	67	goto end;
	68	}
	69
	70	switch (output->type) {
	71	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
	72	ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
	73	if (ret) {
	74	WARN_ON_ONCE(1);
	75	goto end;
	76	}
	77	break;
	78	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
	79	ret = lttng_msgpack_write_signed_integer(writer, output->u.u);
	80	if (ret) {
	81	WARN_ON_ONCE(1);
	82	goto end;
	83	}
	84	break;
	85	default:
	86	WARN_ON(1);
	87	}
	88
	89	ret = lttng_msgpack_end_map(writer);
	90	if (ret)
	91	WARN_ON_ONCE(1);
	92
	93	end:
	94	return ret;
	95	}
	96
	97	static
	98	int64_t capture_sequence_element_signed(uint8_t *ptr,
437d5aa5	99	const struct lttng_kernel_type_integer *type)
42f3ef8c FD	100	{
	101	int64_t value = 0;
	102	unsigned int size = type->size;
	103	bool byte_order_reversed = type->reverse_byte_order;
	104
	105	switch (size) {
	106	case 8:
	107	value = *ptr;
	108	break;
	109	case 16:
	110	{
	111	int16_t tmp;
	112	tmp = (int16_t ) ptr;
	113	if (byte_order_reversed)
	114	__swab16s(&tmp);
	115
	116	value = tmp;
	117	break;
	118	}
	119	case 32:
	120	{
	121	int32_t tmp;
	122	tmp = (int32_t ) ptr;
	123	if (byte_order_reversed)
	124	__swab32s(&tmp);
	125
	126	value = tmp;
	127	break;
	128	}
	129	case 64:
	130	{
	131	int64_t tmp;
	132	tmp = (int64_t ) ptr;
	133	if (byte_order_reversed)
	134	__swab64s(&tmp);
	135
	136	value = tmp;
	137	break;
	138	}
	139	default:
	140	WARN_ON(1);
	141	}
	142
	143	return value;
	144	}
	145
	146	static
	147	uint64_t capture_sequence_element_unsigned(uint8_t *ptr,
437d5aa5	148	const struct lttng_kernel_type_integer *type)
42f3ef8c FD	149	{
	150	uint64_t value = 0;
	151	unsigned int size = type->size;
	152	bool byte_order_reversed = type->reverse_byte_order;
	153
	154	switch (size) {
	155	case 8:
	156	value = *ptr;
	157	break;
	158	case 16:
	159	{
	160	uint16_t tmp;
	161	tmp = (uint16_t ) ptr;
	162	if (byte_order_reversed)
	163	__swab16s(&tmp);
	164
	165	value = tmp;
	166	break;
	167	}
	168	case 32:
	169	{
	170	uint32_t tmp;
	171	tmp = (uint32_t ) ptr;
	172	if (byte_order_reversed)
	173	__swab32s(&tmp);
	174
	175	value = tmp;
	176	break;
	177	}
	178	case 64:
	179	{
	180	uint64_t tmp;
	181	tmp = (uint64_t ) ptr;
	182	if (byte_order_reversed)
	183	__swab64s(&tmp);
	184
	185	value = tmp;
	186	break;
	187	}
	188	default:
	189	WARN_ON(1);
	190	}
	191
	192	return value;
	193	}
	194
42f3ef8c FD	195	int capture_sequence(struct lttng_msgpack_writer *writer,
	196	struct lttng_interpreter_output *output)
	197	{
437d5aa5 MD	198	const struct lttng_kernel_type_integer *integer_type = NULL;
437d5aa5 MD	199	const struct lttng_kernel_type_common *nested_type;
42f3ef8c FD	200	uint8_t *ptr;
	201	bool signedness;
	202	int ret, i;
	203
	204	ret = lttng_msgpack_begin_array(writer, output->u.sequence.nr_elem);
	205	if (ret) {
	206	WARN_ON_ONCE(1);
	207	goto end;
	208	}
	209
	210	ptr = (uint8_t *) output->u.sequence.ptr;
	211	nested_type = output->u.sequence.nested_type;
12bb2edb MD	212	switch (nested_type->type) {
12bb2edb MD	213	case lttng_kernel_type_integer:
437d5aa5	214	integer_type = lttng_kernel_get_type_integer(nested_type);
42f3ef8c	215	break;
437d5aa5	216	case lttng_kernel_type_enum:
42f3ef8c	217	/* Treat enumeration as an integer. */
437d5aa5	218	integer_type = lttng_kernel_get_type_integer(lttng_kernel_get_type_enum(nested_type)->container_type);
42f3ef8c FD	219	break;
	220	default:
	221	/* Capture of array of non-integer are not supported. */
	222	WARN_ON(1);
	223	}
	224	signedness = integer_type->signedness;
	225	for (i = 0; i < output->u.sequence.nr_elem; i++) {
	226	if (signedness) {
	227	ret = lttng_msgpack_write_signed_integer(writer,
	228	capture_sequence_element_signed(ptr, integer_type));
	229	if (ret) {
	230	WARN_ON_ONCE(1);
	231	goto end;
	232	}
	233	} else {
	234	ret = lttng_msgpack_write_unsigned_integer(writer,
	235	capture_sequence_element_unsigned(ptr, integer_type));
	236	if (ret) {
	237	WARN_ON_ONCE(1);
	238	goto end;
	239	}
	240	}
	241
	242	/*
	243	* We assume that alignment is smaller or equal to the size.
	244	* This currently holds true but if it changes in the future,
	245	* we will want to change the pointer arithmetics below to
	246	* take into account that the next element might be further
	247	* away.
	248	*/
	249	WARN_ON(integer_type->alignment > integer_type->size);
	250
	251	/* Size is in number of bits. */
	252	ptr += (integer_type->size / CHAR_BIT) ;
	253	}
	254
	255	ret = lttng_msgpack_end_array(writer);
	256	if (ret)
	257	WARN_ON_ONCE(1);
	258	end:
	259	return ret;
	260	}
	261
99d223ad FD	262	static
	263	int notification_append_capture(
	264	struct lttng_event_notifier_notification *notif,
	265	struct lttng_interpreter_output *output)
	266	{
	267	struct lttng_msgpack_writer *writer = &notif->writer;
	268	int ret = 0;
	269
	270	switch (output->type) {
	271	case LTTNG_INTERPRETER_TYPE_S64:
	272	ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
	273	if (ret) {
	274	WARN_ON_ONCE(1);
	275	goto end;
	276	}
	277	break;
	278	case LTTNG_INTERPRETER_TYPE_U64:
	279	ret = lttng_msgpack_write_unsigned_integer(writer, output->u.u);
	280	if (ret) {
	281	WARN_ON_ONCE(1);
	282	goto end;
	283	}
	284	break;
	285	case LTTNG_INTERPRETER_TYPE_STRING:
8915cf5e MD	286	if (output->u.str.user) {
	287	ret = lttng_msgpack_write_user_str(writer, output->u.str.user_str);
	288	} else {
	289	ret = lttng_msgpack_write_str(writer, output->u.str.str);
	290	}
99d223ad FD	291	if (ret) {
	292	WARN_ON_ONCE(1);
	293	goto end;
	294	}
	295	break;
	296	case LTTNG_INTERPRETER_TYPE_SEQUENCE:
	297	ret = capture_sequence(writer, output);
	298	if (ret) {
	299	WARN_ON_ONCE(1);
	300	goto end;
	301	}
	302	break;
	303	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
	304	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
	305	ret = capture_enum(writer, output);
	306	if (ret) {
	307	WARN_ON_ONCE(1);
	308	goto end;
	309	}
	310	break;
	311	default:
	312	ret = -1;
	313	WARN_ON(1);
	314	}
	315	end:
	316	return ret;
	317	}
	318
	319	static
	320	int notification_append_empty_capture(
	321	struct lttng_event_notifier_notification *notif)
	322	{
	323	int ret = lttng_msgpack_write_nil(&notif->writer);
	324	if (ret)
	325	WARN_ON_ONCE(1);
	326
	327	return ret;
	328	}
	329
	330	static
	331	int notification_init(struct lttng_event_notifier_notification *notif,
a67ba386	332	struct lttng_kernel_event_notifier *event_notifier)
99d223ad FD	333	{
	334	struct lttng_msgpack_writer *writer = &notif->writer;
	335	int ret = 0;
	336
	337	notif->has_captures = false;
	338
a67ba386	339	if (event_notifier->priv->num_captures > 0) {
99d223ad FD	340	lttng_msgpack_writer_init(writer, notif->capture_buf,
	341	CAPTURE_BUFFER_SIZE);
	342
a67ba386	343	ret = lttng_msgpack_begin_array(writer, event_notifier->priv->num_captures);
99d223ad FD	344	if (ret) {
	345	WARN_ON_ONCE(1);
	346	goto end;
	347	}
	348
	349	notif->has_captures = true;
	350	}
	351
	352	end:
	353	return ret;
	354	}
	355
99f52fcc	356	static
a67ba386	357	void record_error(struct lttng_kernel_event_notifier *event_notifier)
99f52fcc FD	358	{
99f52fcc FD	359
a67ba386	360	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
ab04d7b1	361	struct lttng_counter *error_counter;
99f52fcc FD	362	size_t dimension_index[1];
	363	int ret;
	364
ab04d7b1	365	/*
6657edec	366	* lttng_smp_load_acquire paired with lttng_smp_store_release orders
ab04d7b1 MD	367	* creation of the error counter and setting error_counter_len
	368	* before the error_counter is used.
	369	*/
6657edec	370	error_counter = lttng_smp_load_acquire(&event_notifier_group->error_counter);
a83d6831	371	/* This group may not have an error counter attached to it. */
ab04d7b1	372	if (!error_counter)
a83d6831 FD	373	return;
a83d6831 FD	374
a67ba386	375	dimension_index[0] = event_notifier->priv->error_counter_index;
99f52fcc	376
ab04d7b1	377	ret = error_counter->ops->counter_add(error_counter->counter,
99f52fcc FD	378	dimension_index, 1);
	379	if (ret)
	380	WARN_ON_ONCE(1);
	381	}
	382
99d223ad FD	383	static
99d223ad FD	384	void notification_send(struct lttng_event_notifier_notification *notif,
a67ba386	385	struct lttng_kernel_event_notifier *event_notifier)
21f58fb7	386	{
a67ba386	387	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
8a57ec02	388	struct lttng_kernel_ring_buffer_ctx ctx;
606828e4	389	struct lttng_kernel_abi_event_notifier_notification kernel_notif;
99d223ad	390	size_t capture_buffer_content_len, reserve_size;
21f58fb7 FD	391	int ret;
21f58fb7 FD	392
99d223ad	393	reserve_size = sizeof(kernel_notif);
a67ba386	394	kernel_notif.token = event_notifier->priv->parent.user_token;
21f58fb7	395
99d223ad FD	396	if (notif->has_captures) {
	397	capture_buffer_content_len = notif->writer.write_pos - notif->writer.buffer;
	398	} else {
	399	capture_buffer_content_len = 0;
	400	}
	401
	402	WARN_ON_ONCE(capture_buffer_content_len > CAPTURE_BUFFER_SIZE);
	403
	404	reserve_size += capture_buffer_content_len;
	405	kernel_notif.capture_buf_size = capture_buffer_content_len;
	406
8a445457	407	lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, reserve_size,
b1199bd3	408	lttng_alignof(kernel_notif), NULL);
c2fb9c1c	409	ret = event_notifier_group->ops->event_reserve(&ctx);
21f58fb7	410	if (ret < 0) {
99f52fcc	411	record_error(event_notifier);
21f58fb7 FD	412	return;
21f58fb7 FD	413	}
99d223ad	414
99d223ad FD	415	/* Write the notif structure. */
99d223ad FD	416	event_notifier_group->ops->event_write(&ctx, &kernel_notif,
f5ffbd77	417	sizeof(kernel_notif), lttng_alignof(kernel_notif));
99d223ad FD	418
	419	/*
	420	* Write the capture buffer. No need to realigned as the below is a raw
	421	* char* buffer.
	422	*/
	423	event_notifier_group->ops->event_write(&ctx, &notif->capture_buf,
f5ffbd77	424	capture_buffer_content_len, 1);
99d223ad	425
21f58fb7 FD	426	event_notifier_group->ops->event_commit(&ctx);
	427	irq_work_queue(&event_notifier_group->wakeup_pending);
	428	}
99d223ad	429
a67ba386	430	void lttng_event_notifier_notification_send(struct lttng_kernel_event_notifier *event_notifier,
c3eddb2e	431	const char *stack_data,
a92e844e	432	struct lttng_kernel_probe_ctx *probe_ctx,
a67ba386	433	struct lttng_kernel_notification_ctx *notif_ctx)
99d223ad FD	434	{
	435	struct lttng_event_notifier_notification notif = { 0 };
	436	int ret;
	437
a67ba386	438	if (unlikely(!READ_ONCE(event_notifier->parent.enabled)))
99d223ad FD	439	return;
	440
	441	ret = notification_init(&notif, event_notifier);
	442	if (ret) {
	443	WARN_ON_ONCE(1);
	444	goto end;
	445	}
	446
c3eddb2e	447	if (unlikely(notif_ctx->eval_capture)) {
218585b9	448	struct lttng_kernel_bytecode_runtime *capture_bc_runtime;
99d223ad FD	449
	450	/*
	451	* Iterate over all the capture bytecodes. If the interpreter
	452	* functions returns successfully, append the value of the
	453	* `output` parameter to the capture buffer. If the interpreter
	454	* fails, append an empty capture to the buffer.
	455	*/
c3eddb2e	456	list_for_each_entry_rcu(capture_bc_runtime,
a67ba386	457	&event_notifier->priv->capture_bytecode_runtime_head, node) {
99d223ad FD	458	struct lttng_interpreter_output output;
99d223ad FD	459
8a445457 MD	460	if (capture_bc_runtime->interpreter_func(capture_bc_runtime,
8a445457 MD	461	stack_data, probe_ctx, &output) == LTTNG_KERNEL_BYTECODE_INTERPRETER_OK)
99d223ad FD	462	ret = notification_append_capture(&notif, &output);
	463	else
	464	ret = notification_append_empty_capture(&notif);
	465
	466	if (ret)
	467	printk(KERN_WARNING "Error appending capture to notification");
	468	}
	469	}
	470
	471	/*
	472	* Send the notification (including the capture buffer) to the
	473	* sessiond.
	474	*/
	475	notification_send(&notif, event_notifier);
	476	end:
	477	return;
	478	}