[lttng-modules.git] / src / lttng-event-notifier-notification.c

/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
 *
 * lttng-event-notifier-notification.c
 *
 * Copyright (C) 2020 Francis Deslauriers <francis.deslauriers@efficios.com>
 */

#include <linux/bug.h>

#include <lttng/lttng-bytecode.h>
#include <lttng/events.h>
#include <lttng/msgpack.h>
#include <lttng/event-notifier-notification.h>
#include <lttng/events-internal.h>
#include <wrapper/barrier.h>

/*
 * The capture buffer size needs to be below 1024 bytes to avoid the
 * frame to be larger than the 1024 limit enforced by the kernel. If we
 * ever need to increase it, we will need to use a memory allocation
 * scheme which allows allocating temporary memory chunks from the
 * instrumentation sites. This could be done by adapting lttng
 * tp-mempool to become nmi-safe and lock-free.
 */
#define CAPTURE_BUFFER_SIZE 512

struct lttng_event_notifier_notification {
	int notification_fd;
	uint64_t event_notifier_token;
	uint8_t capture_buf[CAPTURE_BUFFER_SIZE];
	struct lttng_msgpack_writer writer;
	bool has_captures;
};

static
int capture_enum(struct lttng_msgpack_writer *writer,
		struct lttng_interpreter_output *output)
{
	int ret;

	/*
	 * Enums are captured as a map containing 2 key-value pairs. Such as:
	 * - type: enum
	 *   value: 177
	 */
	ret = lttng_msgpack_begin_map(writer, 2);
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = lttng_msgpack_write_str(writer, "type");
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = lttng_msgpack_write_str(writer, "enum");
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = lttng_msgpack_write_str(writer, "value");
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	switch (output->type) {
	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
		ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
		ret = lttng_msgpack_write_signed_integer(writer, output->u.u);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	default:
		WARN_ON(1);
	}

	ret = lttng_msgpack_end_map(writer);
	if (ret)
		WARN_ON_ONCE(1);

end:
	return ret;
}

static
int64_t capture_sequence_element_signed(uint8_t *ptr,
		const struct lttng_kernel_type_integer *type)
{
	int64_t value = 0;
	unsigned int size = type->size;
	bool byte_order_reversed = type->reverse_byte_order;

	switch (size) {
	case 8:
		value = *ptr;
		break;
	case 16:
	{
		int16_t tmp;
		tmp = *(int16_t *) ptr;
		if (byte_order_reversed)
			__swab16s(&tmp);

		value = tmp;
		break;
	}
	case 32:
	{
		int32_t tmp;
		tmp = *(int32_t *) ptr;
		if (byte_order_reversed)
			__swab32s(&tmp);

		value = tmp;
		break;
	}
	case 64:
	{
		int64_t tmp;
		tmp = *(int64_t *) ptr;
		if (byte_order_reversed)
			__swab64s(&tmp);

		value = tmp;
		break;
	}
	default:
		WARN_ON(1);
	}

	return value;
}

static
uint64_t capture_sequence_element_unsigned(uint8_t *ptr,
		const struct lttng_kernel_type_integer *type)
{
	uint64_t value = 0;
	unsigned int size = type->size;
	bool byte_order_reversed = type->reverse_byte_order;

	switch (size) {
	case 8:
		value = *ptr;
		break;
	case 16:
	{
		uint16_t tmp;
		tmp = *(uint16_t *) ptr;
		if (byte_order_reversed)
			__swab16s(&tmp);

		value = tmp;
		break;
	}
	case 32:
	{
		uint32_t tmp;
		tmp = *(uint32_t *) ptr;
		if (byte_order_reversed)
			__swab32s(&tmp);

		value = tmp;
		break;
	}
	case 64:
	{
		uint64_t tmp;
		tmp = *(uint64_t *) ptr;
		if (byte_order_reversed)
			__swab64s(&tmp);

		value = tmp;
		break;
	}
	default:
		WARN_ON(1);
	}

	return value;
}

int capture_sequence(struct lttng_msgpack_writer *writer,
		struct lttng_interpreter_output *output)
{
	const struct lttng_kernel_type_integer *integer_type = NULL;
	const struct lttng_kernel_type_common *nested_type;
	uint8_t *ptr;
	bool signedness;
	int ret, i;

	ret = lttng_msgpack_begin_array(writer, output->u.sequence.nr_elem);
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ptr = (uint8_t *) output->u.sequence.ptr;
	nested_type = output->u.sequence.nested_type;
	switch (nested_type->type) {
	case lttng_kernel_type_integer:
		integer_type = lttng_kernel_get_type_integer(nested_type);
		break;
	case lttng_kernel_type_enum:
		/* Treat enumeration as an integer. */
		integer_type = lttng_kernel_get_type_integer(lttng_kernel_get_type_enum(nested_type)->container_type);
		break;
	default:
		/* Capture of array of non-integer are not supported. */
		WARN_ON(1);
	}
	signedness = integer_type->signedness;
	for (i = 0; i < output->u.sequence.nr_elem; i++) {
		if (signedness) {
			ret = lttng_msgpack_write_signed_integer(writer,
				capture_sequence_element_signed(ptr, integer_type));
			if (ret) {
				WARN_ON_ONCE(1);
				goto end;
			}
		} else {
			ret = lttng_msgpack_write_unsigned_integer(writer,
				capture_sequence_element_unsigned(ptr, integer_type));
			if (ret) {
				WARN_ON_ONCE(1);
				goto end;
			}
		}

		/*
		 * We assume that alignment is smaller or equal to the size.
		 * This currently holds true but if it changes in the future,
		 * we will want to change the pointer arithmetics below to
		 * take into account that the next element might be further
		 * away.
		 */
		WARN_ON(integer_type->alignment > integer_type->size);

		/* Size is in number of bits. */
		ptr += (integer_type->size / CHAR_BIT) ;
	}

	ret = lttng_msgpack_end_array(writer);
	if (ret)
		WARN_ON_ONCE(1);
end:
	return ret;
}

static
int notification_append_capture(
		struct lttng_event_notifier_notification *notif,
		struct lttng_interpreter_output *output)
{
	struct lttng_msgpack_writer *writer = &notif->writer;
	int ret = 0;

	switch (output->type) {
	case LTTNG_INTERPRETER_TYPE_S64:
		ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_U64:
		ret = lttng_msgpack_write_unsigned_integer(writer, output->u.u);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_STRING:
		ret = lttng_msgpack_write_str(writer, output->u.str.str);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_SEQUENCE:
		ret = capture_sequence(writer, output);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
		ret = capture_enum(writer, output);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}
		break;
	default:
		ret = -1;
		WARN_ON(1);
	}
end:
	return ret;
}

static
int notification_append_empty_capture(
		struct lttng_event_notifier_notification *notif)
{
	int ret = lttng_msgpack_write_nil(&notif->writer);
	if (ret)
		WARN_ON_ONCE(1);

	return ret;
}

static
int notification_init(struct lttng_event_notifier_notification *notif,
		struct lttng_kernel_event_notifier *event_notifier)
{
	struct lttng_msgpack_writer *writer = &notif->writer;
	int ret = 0;

	notif->has_captures = false;

	if (event_notifier->priv->num_captures > 0) {
		lttng_msgpack_writer_init(writer, notif->capture_buf,
				CAPTURE_BUFFER_SIZE);

		ret = lttng_msgpack_begin_array(writer, event_notifier->priv->num_captures);
		if (ret) {
			WARN_ON_ONCE(1);
			goto end;
		}

		notif->has_captures = true;
	}

end:
	return ret;
}

static
void record_error(struct lttng_kernel_event_notifier *event_notifier)
{

	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
	struct lttng_counter *error_counter;
	size_t dimension_index[1];
	int ret;

	/*
	 * lttng_smp_load_acquire paired with lttng_smp_store_release orders
	 * creation of the error counter and setting error_counter_len
	 * before the error_counter is used.
	 */
	error_counter = lttng_smp_load_acquire(&event_notifier_group->error_counter);
	/* This group may not have an error counter attached to it. */
	if (!error_counter)
		return;

	dimension_index[0] = event_notifier->priv->error_counter_index;

	ret = error_counter->ops->counter_add(error_counter->counter,
			dimension_index, 1);
	if (ret)
		WARN_ON_ONCE(1);
}

static
void notification_send(struct lttng_event_notifier_notification *notif,
		struct lttng_kernel_event_notifier *event_notifier)
{
	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
	struct lttng_kernel_ring_buffer_ctx ctx;
	struct lttng_kernel_abi_event_notifier_notification kernel_notif;
	size_t capture_buffer_content_len, reserve_size;
	int ret;

	reserve_size = sizeof(kernel_notif);
	kernel_notif.token = event_notifier->priv->parent.user_token;

	if (notif->has_captures) {
		capture_buffer_content_len = notif->writer.write_pos - notif->writer.buffer;
	} else {
		capture_buffer_content_len = 0;
	}

	WARN_ON_ONCE(capture_buffer_content_len > CAPTURE_BUFFER_SIZE);

	reserve_size += capture_buffer_content_len;
	kernel_notif.capture_buf_size = capture_buffer_content_len;

	lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, reserve_size,
			lttng_alignof(kernel_notif), NULL);
	ret = event_notifier_group->ops->event_reserve(&ctx);
	if (ret < 0) {
		record_error(event_notifier);
		return;
	}

	/* Write the notif structure. */
	event_notifier_group->ops->event_write(&ctx, &kernel_notif,
			sizeof(kernel_notif), lttng_alignof(kernel_notif));

	/*
	 * Write the capture buffer. No need to realigned as the below is a raw
	 * char* buffer.
	 */
	event_notifier_group->ops->event_write(&ctx, &notif->capture_buf,
			capture_buffer_content_len, 1);

	event_notifier_group->ops->event_commit(&ctx);
	irq_work_queue(&event_notifier_group->wakeup_pending);
}

void lttng_event_notifier_notification_send(struct lttng_kernel_event_notifier *event_notifier,
		const char *stack_data,
		struct lttng_kernel_probe_ctx *probe_ctx,
		struct lttng_kernel_notification_ctx *notif_ctx)
{
	struct lttng_event_notifier_notification notif = { 0 };
	int ret;

	if (unlikely(!READ_ONCE(event_notifier->parent.enabled)))
		return;

	ret = notification_init(&notif, event_notifier);
	if (ret) {
		WARN_ON_ONCE(1);
		goto end;
	}

	if (unlikely(notif_ctx->eval_capture)) {
		struct lttng_kernel_bytecode_runtime *capture_bc_runtime;

		/*
		 * Iterate over all the capture bytecodes. If the interpreter
		 * functions returns successfully, append the value of the
		 * `output` parameter to the capture buffer. If the interpreter
		 * fails, append an empty capture to the buffer.
		 */
		list_for_each_entry_rcu(capture_bc_runtime,
				&event_notifier->priv->capture_bytecode_runtime_head, node) {
			struct lttng_interpreter_output output;

			if (capture_bc_runtime->interpreter_func(capture_bc_runtime,
					stack_data, probe_ctx, &output) == LTTNG_KERNEL_BYTECODE_INTERPRETER_OK)
				ret = notification_append_capture(&notif, &output);
			else
				ret = notification_append_empty_capture(&notif);

			if (ret)
				printk(KERN_WARNING "Error appending capture to notification");
		}
	}

	/*
	 * Send the notification (including the capture buffer) to the
	 * sessiond.
	 */
	notification_send(&notif, event_notifier);
end:
	return;
}
Commit	Line	Data
21f58fb7 FD	1	/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
	2	*
	3	* lttng-event-notifier-notification.c
	4	*
	5	* Copyright (C) 2020 Francis Deslauriers <francis.deslauriers@efficios.com>
	6	*/
	7
42f3ef8c FD	8	#include <linux/bug.h>
	9
	10	#include <lttng/lttng-bytecode.h>
21f58fb7	11	#include <lttng/events.h>
42f3ef8c	12	#include <lttng/msgpack.h>
21f58fb7	13	#include <lttng/event-notifier-notification.h>
437d5aa5	14	#include <lttng/events-internal.h>
6657edec	15	#include <wrapper/barrier.h>
21f58fb7	16
99d223ad	17	/*
e699ee6c MD	18	* The capture buffer size needs to be below 1024 bytes to avoid the
	19	* frame to be larger than the 1024 limit enforced by the kernel. If we
	20	* ever need to increase it, we will need to use a memory allocation
	21	* scheme which allows allocating temporary memory chunks from the
	22	* instrumentation sites. This could be done by adapting lttng
	23	* tp-mempool to become nmi-safe and lock-free.
99d223ad FD	24	*/
	25	#define CAPTURE_BUFFER_SIZE 512
	26
	27	struct lttng_event_notifier_notification {
	28	int notification_fd;
	29	uint64_t event_notifier_token;
	30	uint8_t capture_buf[CAPTURE_BUFFER_SIZE];
	31	struct lttng_msgpack_writer writer;
	32	bool has_captures;
	33	};
	34
42f3ef8c FD	35	static
	36	int capture_enum(struct lttng_msgpack_writer *writer,
	37	struct lttng_interpreter_output *output)
	38	{
	39	int ret;
	40
	41	/*
	42	* Enums are captured as a map containing 2 key-value pairs. Such as:
	43	* - type: enum
	44	* value: 177
	45	*/
	46	ret = lttng_msgpack_begin_map(writer, 2);
	47	if (ret) {
	48	WARN_ON_ONCE(1);
	49	goto end;
	50	}
	51
	52	ret = lttng_msgpack_write_str(writer, "type");
	53	if (ret) {
	54	WARN_ON_ONCE(1);
	55	goto end;
	56	}
	57
	58	ret = lttng_msgpack_write_str(writer, "enum");
	59	if (ret) {
	60	WARN_ON_ONCE(1);
	61	goto end;
	62	}
	63
	64	ret = lttng_msgpack_write_str(writer, "value");
	65	if (ret) {
	66	WARN_ON_ONCE(1);
	67	goto end;
	68	}
	69
	70	switch (output->type) {
	71	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
	72	ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
	73	if (ret) {
	74	WARN_ON_ONCE(1);
	75	goto end;
	76	}
	77	break;
	78	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
	79	ret = lttng_msgpack_write_signed_integer(writer, output->u.u);
	80	if (ret) {
	81	WARN_ON_ONCE(1);
	82	goto end;
	83	}
	84	break;
	85	default:
	86	WARN_ON(1);
	87	}
	88
	89	ret = lttng_msgpack_end_map(writer);
	90	if (ret)
	91	WARN_ON_ONCE(1);
	92
	93	end:
	94	return ret;
	95	}
	96
	97	static
	98	int64_t capture_sequence_element_signed(uint8_t *ptr,
437d5aa5	99	const struct lttng_kernel_type_integer *type)
42f3ef8c FD	100	{
	101	int64_t value = 0;
	102	unsigned int size = type->size;
	103	bool byte_order_reversed = type->reverse_byte_order;
	104
	105	switch (size) {
	106	case 8:
	107	value = *ptr;
	108	break;
	109	case 16:
	110	{
	111	int16_t tmp;
	112	tmp = (int16_t ) ptr;
	113	if (byte_order_reversed)
	114	__swab16s(&tmp);
	115
	116	value = tmp;
	117	break;
	118	}
	119	case 32:
	120	{
	121	int32_t tmp;
	122	tmp = (int32_t ) ptr;
	123	if (byte_order_reversed)
	124	__swab32s(&tmp);
	125
	126	value = tmp;
	127	break;
	128	}
	129	case 64:
	130	{
	131	int64_t tmp;
	132	tmp = (int64_t ) ptr;
	133	if (byte_order_reversed)
	134	__swab64s(&tmp);
	135
	136	value = tmp;
	137	break;
	138	}
	139	default:
	140	WARN_ON(1);
	141	}
	142
	143	return value;
	144	}
	145
	146	static
	147	uint64_t capture_sequence_element_unsigned(uint8_t *ptr,
437d5aa5	148	const struct lttng_kernel_type_integer *type)
42f3ef8c FD	149	{
	150	uint64_t value = 0;
	151	unsigned int size = type->size;
	152	bool byte_order_reversed = type->reverse_byte_order;
	153
	154	switch (size) {
	155	case 8:
	156	value = *ptr;
	157	break;
	158	case 16:
	159	{
	160	uint16_t tmp;
	161	tmp = (uint16_t ) ptr;
	162	if (byte_order_reversed)
	163	__swab16s(&tmp);
	164
	165	value = tmp;
	166	break;
	167	}
	168	case 32:
	169	{
	170	uint32_t tmp;
	171	tmp = (uint32_t ) ptr;
	172	if (byte_order_reversed)
	173	__swab32s(&tmp);
	174
	175	value = tmp;
	176	break;
	177	}
	178	case 64:
	179	{
	180	uint64_t tmp;
	181	tmp = (uint64_t ) ptr;
	182	if (byte_order_reversed)
	183	__swab64s(&tmp);
	184
	185	value = tmp;
	186	break;
	187	}
	188	default:
	189	WARN_ON(1);
	190	}
	191
	192	return value;
	193	}
	194
42f3ef8c FD	195	int capture_sequence(struct lttng_msgpack_writer *writer,
	196	struct lttng_interpreter_output *output)
	197	{
437d5aa5 MD	198	const struct lttng_kernel_type_integer *integer_type = NULL;
437d5aa5 MD	199	const struct lttng_kernel_type_common *nested_type;
42f3ef8c FD	200	uint8_t *ptr;
	201	bool signedness;
	202	int ret, i;
	203
	204	ret = lttng_msgpack_begin_array(writer, output->u.sequence.nr_elem);
	205	if (ret) {
	206	WARN_ON_ONCE(1);
	207	goto end;
	208	}
	209
	210	ptr = (uint8_t *) output->u.sequence.ptr;
	211	nested_type = output->u.sequence.nested_type;
12bb2edb MD	212	switch (nested_type->type) {
12bb2edb MD	213	case lttng_kernel_type_integer:
437d5aa5	214	integer_type = lttng_kernel_get_type_integer(nested_type);
42f3ef8c	215	break;
437d5aa5	216	case lttng_kernel_type_enum:
42f3ef8c	217	/* Treat enumeration as an integer. */
437d5aa5	218	integer_type = lttng_kernel_get_type_integer(lttng_kernel_get_type_enum(nested_type)->container_type);
42f3ef8c FD	219	break;
	220	default:
	221	/* Capture of array of non-integer are not supported. */
	222	WARN_ON(1);
	223	}
	224	signedness = integer_type->signedness;
	225	for (i = 0; i < output->u.sequence.nr_elem; i++) {
	226	if (signedness) {
	227	ret = lttng_msgpack_write_signed_integer(writer,
	228	capture_sequence_element_signed(ptr, integer_type));
	229	if (ret) {
	230	WARN_ON_ONCE(1);
	231	goto end;
	232	}
	233	} else {
	234	ret = lttng_msgpack_write_unsigned_integer(writer,
	235	capture_sequence_element_unsigned(ptr, integer_type));
	236	if (ret) {
	237	WARN_ON_ONCE(1);
	238	goto end;
	239	}
	240	}
	241
	242	/*
	243	* We assume that alignment is smaller or equal to the size.
	244	* This currently holds true but if it changes in the future,
	245	* we will want to change the pointer arithmetics below to
	246	* take into account that the next element might be further
	247	* away.
	248	*/
	249	WARN_ON(integer_type->alignment > integer_type->size);
	250
	251	/* Size is in number of bits. */
	252	ptr += (integer_type->size / CHAR_BIT) ;
	253	}
	254
	255	ret = lttng_msgpack_end_array(writer);
	256	if (ret)
	257	WARN_ON_ONCE(1);
	258	end:
	259	return ret;
	260	}
	261
99d223ad FD	262	static
	263	int notification_append_capture(
	264	struct lttng_event_notifier_notification *notif,
	265	struct lttng_interpreter_output *output)
	266	{
	267	struct lttng_msgpack_writer *writer = &notif->writer;
	268	int ret = 0;
	269
	270	switch (output->type) {
	271	case LTTNG_INTERPRETER_TYPE_S64:
	272	ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
	273	if (ret) {
	274	WARN_ON_ONCE(1);
	275	goto end;
	276	}
	277	break;
	278	case LTTNG_INTERPRETER_TYPE_U64:
	279	ret = lttng_msgpack_write_unsigned_integer(writer, output->u.u);
	280	if (ret) {
	281	WARN_ON_ONCE(1);
	282	goto end;
	283	}
	284	break;
	285	case LTTNG_INTERPRETER_TYPE_STRING:
	286	ret = lttng_msgpack_write_str(writer, output->u.str.str);
	287	if (ret) {
	288	WARN_ON_ONCE(1);
	289	goto end;
	290	}
	291	break;
	292	case LTTNG_INTERPRETER_TYPE_SEQUENCE:
	293	ret = capture_sequence(writer, output);
	294	if (ret) {
	295	WARN_ON_ONCE(1);
	296	goto end;
	297	}
	298	break;
	299	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
	300	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
	301	ret = capture_enum(writer, output);
	302	if (ret) {
	303	WARN_ON_ONCE(1);
	304	goto end;
	305	}
	306	break;
	307	default:
	308	ret = -1;
	309	WARN_ON(1);
	310	}
	311	end:
	312	return ret;
	313	}
	314
	315	static
	316	int notification_append_empty_capture(
	317	struct lttng_event_notifier_notification *notif)
	318	{
	319	int ret = lttng_msgpack_write_nil(&notif->writer);
	320	if (ret)
	321	WARN_ON_ONCE(1);
	322
	323	return ret;
	324	}
	325
326	static
327	int notification_init(struct lttng_event_notifier_notification *notif,
a67ba386	328	struct lttng_kernel_event_notifier *event_notifier)
99d223ad FD	329	{
	330	struct lttng_msgpack_writer *writer = &notif->writer;
	331	int ret = 0;
	332
	333	notif->has_captures = false;
	334
a67ba386	335	if (event_notifier->priv->num_captures > 0) {
99d223ad FD	336	lttng_msgpack_writer_init(writer, notif->capture_buf,
	337	CAPTURE_BUFFER_SIZE);
	338
a67ba386	339	ret = lttng_msgpack_begin_array(writer, event_notifier->priv->num_captures);
99d223ad FD	340	if (ret) {
	341	WARN_ON_ONCE(1);
	342	goto end;
	343	}
	344
	345	notif->has_captures = true;
	346	}
	347
	348	end:
	349	return ret;
	350	}
	351
99f52fcc	352	static
a67ba386	353	void record_error(struct lttng_kernel_event_notifier *event_notifier)
99f52fcc FD	354	{
99f52fcc FD	355
a67ba386	356	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
ab04d7b1	357	struct lttng_counter *error_counter;
99f52fcc FD	358	size_t dimension_index[1];
	359	int ret;
	360
ab04d7b1	361	/*
6657edec	362	* lttng_smp_load_acquire paired with lttng_smp_store_release orders
ab04d7b1 MD	363	* creation of the error counter and setting error_counter_len
	364	* before the error_counter is used.
	365	*/
6657edec	366	error_counter = lttng_smp_load_acquire(&event_notifier_group->error_counter);
a83d6831	367	/* This group may not have an error counter attached to it. */
ab04d7b1	368	if (!error_counter)
a83d6831 FD	369	return;
a83d6831 FD	370
a67ba386	371	dimension_index[0] = event_notifier->priv->error_counter_index;
99f52fcc	372
ab04d7b1	373	ret = error_counter->ops->counter_add(error_counter->counter,
99f52fcc FD	374	dimension_index, 1);
	375	if (ret)
	376	WARN_ON_ONCE(1);
	377	}
	378
99d223ad FD	379	static
99d223ad FD	380	void notification_send(struct lttng_event_notifier_notification *notif,
a67ba386	381	struct lttng_kernel_event_notifier *event_notifier)
21f58fb7	382	{
a67ba386	383	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
8a57ec02	384	struct lttng_kernel_ring_buffer_ctx ctx;
606828e4	385	struct lttng_kernel_abi_event_notifier_notification kernel_notif;
99d223ad	386	size_t capture_buffer_content_len, reserve_size;
21f58fb7 FD	387	int ret;
21f58fb7 FD	388
99d223ad	389	reserve_size = sizeof(kernel_notif);
a67ba386	390	kernel_notif.token = event_notifier->priv->parent.user_token;
21f58fb7	391
99d223ad FD	392	if (notif->has_captures) {
	393	capture_buffer_content_len = notif->writer.write_pos - notif->writer.buffer;
	394	} else {
	395	capture_buffer_content_len = 0;
	396	}
	397
	398	WARN_ON_ONCE(capture_buffer_content_len > CAPTURE_BUFFER_SIZE);
	399
	400	reserve_size += capture_buffer_content_len;
	401	kernel_notif.capture_buf_size = capture_buffer_content_len;
	402
8a445457	403	lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, reserve_size,
b1199bd3	404	lttng_alignof(kernel_notif), NULL);
c2fb9c1c	405	ret = event_notifier_group->ops->event_reserve(&ctx);
21f58fb7	406	if (ret < 0) {
99f52fcc	407	record_error(event_notifier);
21f58fb7 FD	408	return;
21f58fb7 FD	409	}
99d223ad	410
99d223ad FD	411	/* Write the notif structure. */
99d223ad FD	412	event_notifier_group->ops->event_write(&ctx, &kernel_notif,
f5ffbd77	413	sizeof(kernel_notif), lttng_alignof(kernel_notif));
99d223ad FD	414
	415	/*
	416	* Write the capture buffer. No need to realigned as the below is a raw
	417	* char* buffer.
	418	*/
	419	event_notifier_group->ops->event_write(&ctx, &notif->capture_buf,
f5ffbd77	420	capture_buffer_content_len, 1);
99d223ad	421
21f58fb7 FD	422	event_notifier_group->ops->event_commit(&ctx);
	423	irq_work_queue(&event_notifier_group->wakeup_pending);
	424	}
99d223ad	425
a67ba386	426	void lttng_event_notifier_notification_send(struct lttng_kernel_event_notifier *event_notifier,
c3eddb2e	427	const char *stack_data,
a92e844e	428	struct lttng_kernel_probe_ctx *probe_ctx,
a67ba386	429	struct lttng_kernel_notification_ctx *notif_ctx)
99d223ad FD	430	{
	431	struct lttng_event_notifier_notification notif = { 0 };
	432	int ret;
	433
a67ba386	434	if (unlikely(!READ_ONCE(event_notifier->parent.enabled)))
99d223ad FD	435	return;
	436
	437	ret = notification_init(&notif, event_notifier);
	438	if (ret) {
	439	WARN_ON_ONCE(1);
	440	goto end;
	441	}
	442
c3eddb2e	443	if (unlikely(notif_ctx->eval_capture)) {
218585b9	444	struct lttng_kernel_bytecode_runtime *capture_bc_runtime;
99d223ad FD	445
	446	/*
	447	* Iterate over all the capture bytecodes. If the interpreter
	448	* functions returns successfully, append the value of the
	449	* `output` parameter to the capture buffer. If the interpreter
	450	* fails, append an empty capture to the buffer.
	451	*/
c3eddb2e	452	list_for_each_entry_rcu(capture_bc_runtime,
a67ba386	453	&event_notifier->priv->capture_bytecode_runtime_head, node) {
99d223ad FD	454	struct lttng_interpreter_output output;
99d223ad FD	455
8a445457 MD	456	if (capture_bc_runtime->interpreter_func(capture_bc_runtime,
8a445457 MD	457	stack_data, probe_ctx, &output) == LTTNG_KERNEL_BYTECODE_INTERPRETER_OK)
99d223ad FD	458	ret = notification_append_capture(&notif, &output);
	459	else
	460	ret = notification_append_empty_capture(&notif);
	461
	462	if (ret)
	463	printk(KERN_WARNING "Error appending capture to notification");
	464	}
	465	}
	466
	467	/*
	468	* Send the notification (including the capture buffer) to the
	469	* sessiond.
	470	*/
	471	notification_send(&notif, event_notifier);
	472	end:
	473	return;
	474	}