[lttng-modules.git] / src / lttng-event-notifier-notification.c

/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
 *
 * lttng-event-notifier-notification.c
 *
 * Copyright (C) 2020 Francis Deslauriers <francis.deslauriers@efficios.com>
 */

#include <asm/barrier.h>
#include <linux/bug.h>

#include <lttng/lttng-bytecode.h>
#include <lttng/events.h>
#include <lttng/msgpack.h>
#include <lttng/event-notifier-notification.h>
#include <lttng/events-internal.h>
#include <lttng/probe-user.h>

/*
 * The capture buffer size needs to be below 1024 bytes to avoid the
 * frame to be larger than the 1024 limit enforced by the kernel. If we
 * ever need to increase it, we will need to use a memory allocation
 * scheme which allows allocating temporary memory chunks from the
 * instrumentation sites. This could be done by adapting lttng
 * tp-mempool to become nmi-safe and lock-free.
 */
#define CAPTURE_BUFFER_SIZE 512

#define MSG_WRITE_NIL_LEN 1

struct lttng_event_notifier_notification {
	int notification_fd;
	uint64_t event_notifier_token;
	uint8_t capture_buf[CAPTURE_BUFFER_SIZE];
	struct lttng_msgpack_writer writer;
	bool has_captures;
};

static
int capture_enum(struct lttng_msgpack_writer *writer,
		struct lttng_interpreter_output *output)
{
	int ret;

	/*
	 * Enums are captured as a map containing 2 key-value pairs. Such as:
	 * - type: enum
	 *   value: 177
	 */
	ret = lttng_msgpack_begin_map(writer, 2);
	if (ret) {
		goto end;
	}

	ret = lttng_msgpack_write_str(writer, "type");
	if (ret) {
		goto end;
	}

	ret = lttng_msgpack_write_str(writer, "enum");
	if (ret) {
		goto end;
	}

	ret = lttng_msgpack_write_str(writer, "value");
	if (ret) {
		goto end;
	}

	switch (output->type) {
	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
		ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
		if (ret) {
			goto end;
		}
		break;
	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
		ret = lttng_msgpack_write_signed_integer(writer, output->u.u);
		if (ret) {
			goto end;
		}
		break;
	default:
		WARN_ON_ONCE(1);
		ret = -1;
		goto end;
	}

	ret = lttng_msgpack_end_map(writer);
end:
	return ret;
}

static
int capture_sequence_element_signed(uint8_t *ptr,
		const struct lttng_kernel_type_integer *type,
		int64_t *_value)
{
	int64_t value = 0;
	unsigned int size = type->size;
	bool user = type->user;
	bool byte_order_reversed = type->reverse_byte_order;

	switch (size) {
	case 8:
	{
		int8_t tmp;

		if (user) {
			if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int8_t)))
				return -1;
		} else {
			tmp = *ptr;
		}
		value = tmp;
		break;
	}
	case 16:
	{
		int16_t tmp;

		if (user) {
			if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int16_t)))
				return -1;
		} else {
			tmp = *(int16_t *) ptr;
		}
		if (byte_order_reversed)
			__swab16s(&tmp);
		value = tmp;
		break;
	}
	case 32:
	{
		int32_t tmp;

		if (user) {
			if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int32_t)))
				return -1;
		} else {
			tmp = *(int32_t *) ptr;
		}
		if (byte_order_reversed)
			__swab32s(&tmp);
		value = tmp;
		break;
	}
	case 64:
	{
		int64_t tmp;

		if (user) {
			if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int64_t)))
				return -1;
		} else {
			tmp = *(int64_t *) ptr;
		}
		if (byte_order_reversed)
			__swab64s(&tmp);
		value = tmp;
		break;
	}
	default:
		WARN_ON_ONCE(1);
		return -1;
	}

	*_value = value;
	return 0;
}

static
int capture_sequence_element_unsigned(uint8_t *ptr,
		const struct lttng_kernel_type_integer *type,
		uint64_t *_value)
{
	uint64_t value = 0;
	unsigned int size = type->size;
	bool user = type->user;
	bool byte_order_reversed = type->reverse_byte_order;

	switch (size) {
	case 8:
	{
		uint8_t tmp;

		if (user) {
			if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint8_t)))
				return -1;
		} else {
			tmp = *ptr;
		}
		value = tmp;
		break;
	}
	case 16:
	{
		uint16_t tmp;

		if (user) {
			if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint16_t)))
				return -1;
		} else {
			tmp = *(uint16_t *) ptr;
		}
		if (byte_order_reversed)
			__swab16s(&tmp);
		value = tmp;
		break;
	}
	case 32:
	{
		uint32_t tmp;

		if (user) {
			if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint32_t)))
				return -1;
		} else {
			tmp = *(uint32_t *) ptr;
		}
		if (byte_order_reversed)
			__swab32s(&tmp);
		value = tmp;
		break;
	}
	case 64:
	{
		uint64_t tmp;

		if (user) {
			if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint64_t)))
				return -1;
		} else {
			tmp = *(uint64_t *) ptr;
		}
		if (byte_order_reversed)
			__swab64s(&tmp);
		value = tmp;
		break;
	}
	default:
		WARN_ON_ONCE(1);
		return -1;
	}

	*_value = value;
	return 0;
}

static
int capture_sequence(struct lttng_msgpack_writer *writer,
		struct lttng_interpreter_output *output)
{
	const struct lttng_kernel_type_integer *integer_type = NULL;
	const struct lttng_kernel_type_common *nested_type;
	uint8_t *ptr;
	bool signedness;
	int ret, i;

	ret = lttng_msgpack_begin_array(writer, output->u.sequence.nr_elem);
	if (ret) {
		goto end;
	}

	ptr = (uint8_t *) output->u.sequence.ptr;
	nested_type = output->u.sequence.nested_type;
	switch (nested_type->type) {
	case lttng_kernel_type_integer:
		integer_type = lttng_kernel_get_type_integer(nested_type);
		break;
	case lttng_kernel_type_enum:
		/* Treat enumeration as an integer. */
		integer_type = lttng_kernel_get_type_integer(lttng_kernel_get_type_enum(nested_type)->container_type);
		break;
	default:
		/* Capture of array of non-integer are not supported. */
		WARN_ON_ONCE(1);
		ret = -1;
		goto end;
	}
	signedness = integer_type->signedness;
	for (i = 0; i < output->u.sequence.nr_elem; i++) {
		if (signedness) {
			int64_t v;

			ret = capture_sequence_element_signed(ptr, integer_type, &v);
			if (ret) {
				goto end;
			}
			ret = lttng_msgpack_write_signed_integer(writer, v);
			if (ret) {
				goto end;
			}
		} else {
			uint64_t v;

			ret = capture_sequence_element_unsigned(ptr, integer_type, &v);
			if (ret) {
				goto end;
			}
			ret = lttng_msgpack_write_unsigned_integer(writer, v);
			if (ret) {
				goto end;
			}
		}

		/*
		 * We assume that alignment is smaller or equal to the size.
		 * This currently holds true but if it changes in the future,
		 * we will want to change the pointer arithmetics below to
		 * take into account that the next element might be further
		 * away.
		 */
		WARN_ON_ONCE(integer_type->alignment > integer_type->size);

		/* Size is in number of bits. */
		ptr += (integer_type->size / CHAR_BIT) ;
	}

	ret = lttng_msgpack_end_array(writer);
end:
	return ret;
}

static
int notification_append_capture(
		struct lttng_event_notifier_notification *notif,
		struct lttng_interpreter_output *output)
{
	struct lttng_msgpack_writer *writer = &notif->writer;
	int ret = 0;

	switch (output->type) {
	case LTTNG_INTERPRETER_TYPE_S64:
		ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
		break;
	case LTTNG_INTERPRETER_TYPE_U64:
		ret = lttng_msgpack_write_unsigned_integer(writer, output->u.u);
		break;
	case LTTNG_INTERPRETER_TYPE_STRING:
		if (output->u.str.user) {
			ret = lttng_msgpack_write_user_str(writer, output->u.str.user_str);
		} else {
			ret = lttng_msgpack_write_str(writer, output->u.str.str);
		}
		break;
	case LTTNG_INTERPRETER_TYPE_SEQUENCE:
		ret = capture_sequence(writer, output);
		break;
	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
		ret = capture_enum(writer, output);
		break;
	default:
		ret = -1;
		WARN_ON_ONCE(1);
	}
	return ret;
}

static
int notification_append_empty_capture(
		struct lttng_event_notifier_notification *notif)
{
	return lttng_msgpack_write_nil(&notif->writer);
}

static
int notification_init(struct lttng_event_notifier_notification *notif,
		struct lttng_kernel_event_notifier *event_notifier)
{
	struct lttng_msgpack_writer *writer = &notif->writer;
	int ret = 0;

	notif->has_captures = false;

	if (event_notifier->priv->num_captures > 0) {
		lttng_msgpack_writer_init(writer, notif->capture_buf,
				CAPTURE_BUFFER_SIZE);

		ret = lttng_msgpack_begin_array(writer, event_notifier->priv->num_captures);
		if (ret) {
			goto end;
		}

		notif->has_captures = true;
	}

end:
	return ret;
}

static
void record_error(struct lttng_kernel_event_notifier *event_notifier)
{

	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
	struct lttng_counter *error_counter;
	size_t dimension_index[1];
	int ret;

	/*
	 * smp_load_acquire paired with smp_store_release orders creation of
	 * the error counter and setting error_counter_len before the
	 * error_counter is used.
	 */
	error_counter = smp_load_acquire(&event_notifier_group->error_counter);
	/* This group may not have an error counter attached to it. */
	if (!error_counter)
		return;

	dimension_index[0] = event_notifier->priv->error_counter_index;

	ret = error_counter->ops->counter_add(error_counter->counter,
			dimension_index, 1);
	if (ret)
		WARN_ON_ONCE(1);
}

static
void notification_send(struct lttng_event_notifier_notification *notif,
		struct lttng_kernel_event_notifier *event_notifier)
{
	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
	struct lttng_kernel_ring_buffer_ctx ctx;
	struct lttng_kernel_abi_event_notifier_notification kernel_notif;
	size_t capture_buffer_content_len, reserve_size;
	int ret;

	reserve_size = sizeof(kernel_notif);
	kernel_notif.token = event_notifier->priv->parent.user_token;

	if (notif->has_captures) {
		capture_buffer_content_len = notif->writer.write_pos - notif->writer.buffer;
	} else {
		capture_buffer_content_len = 0;
	}

	WARN_ON_ONCE(capture_buffer_content_len > CAPTURE_BUFFER_SIZE);

	reserve_size += capture_buffer_content_len;
	kernel_notif.capture_buf_size = capture_buffer_content_len;

	lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, reserve_size,
			lttng_alignof(kernel_notif), NULL);
	ret = event_notifier_group->ops->event_reserve(&ctx);
	if (ret < 0) {
		record_error(event_notifier);
		return;
	}

	/* Write the notif structure. */
	event_notifier_group->ops->event_write(&ctx, &kernel_notif,
			sizeof(kernel_notif), lttng_alignof(kernel_notif));

	/*
	 * Write the capture buffer. No need to realigned as the below is a raw
	 * char* buffer.
	 */
	event_notifier_group->ops->event_write(&ctx, &notif->capture_buf,
			capture_buffer_content_len, 1);

	event_notifier_group->ops->event_commit(&ctx);
	irq_work_queue(&event_notifier_group->wakeup_pending);
}

/*
 * Validate that the buffer has enough room to hold empty capture fields.
 */
static
bool validate_buffer_len(struct lttng_event_notifier_notification *notif, size_t captures_left)
{
	if (notif->writer.end_write_pos - notif->writer.write_pos < MSG_WRITE_NIL_LEN * captures_left)
		return false;
	return true;
}

void lttng_event_notifier_notification_send(struct lttng_kernel_event_notifier *event_notifier,
		const char *stack_data,
		struct lttng_kernel_probe_ctx *probe_ctx,
		struct lttng_kernel_notification_ctx *notif_ctx)
{
	struct lttng_event_notifier_notification notif = { 0 };
	size_t captures_left;

	if (notification_init(&notif, event_notifier))
		goto error;

	captures_left = event_notifier->priv->num_captures;
	if (!validate_buffer_len(&notif, captures_left))
		goto error;

	if (unlikely(notif_ctx->eval_capture)) {
		struct lttng_kernel_bytecode_runtime *capture_bc_runtime;

		/*
		 * Iterate over all the capture bytecodes. If the interpreter
		 * functions returns successfully, append the value of the
		 * `output` parameter to the capture buffer. If the interpreter
		 * fails, append an empty capture to the buffer.
		 */
		list_for_each_entry_rcu(capture_bc_runtime,
				&event_notifier->priv->capture_bytecode_runtime_head, node) {
			struct lttng_interpreter_output output;
			uint8_t *save_pos;
			int ret = -1;

			lttng_msgpack_save_writer_pos(&notif.writer, &save_pos);
			captures_left--;
			if (capture_bc_runtime->interpreter_func(capture_bc_runtime,
					stack_data, probe_ctx, &output) == LTTNG_KERNEL_BYTECODE_INTERPRETER_OK)
				ret = notification_append_capture(&notif, &output);
			if (ret || !validate_buffer_len(&notif, captures_left)) {
				/*
				 * On append capture error or if the generated
				 * buffer data would not leave enough room to
				 * write empty capture fields for the remaining
				 * fields, skip the field capture by restoring
				 * the msgpack writer position and writing an
				 * empty capture field.
				 */
				lttng_msgpack_restore_writer_pos(&notif.writer, save_pos);
				ret = notification_append_empty_capture(&notif);
				WARN_ON_ONCE(ret);
			}
		}
	}

	if (notif.has_captures && lttng_msgpack_end_array(&notif.writer))
		goto error;

	/*
	 * Send the notification (including the capture buffer) to the
	 * sessiond.
	 */
	notification_send(&notif, event_notifier);
	return;

error:
	record_error(event_notifier);
	return;
}
Commit	Line	Data
21f58fb7 FD	1	/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
	2	*
	3	* lttng-event-notifier-notification.c
	4	*
	5	* Copyright (C) 2020 Francis Deslauriers <francis.deslauriers@efficios.com>
	6	*/
	7
a27c8df7	8	#include <asm/barrier.h>
42f3ef8c FD	9	#include <linux/bug.h>
	10
	11	#include <lttng/lttng-bytecode.h>
21f58fb7	12	#include <lttng/events.h>
42f3ef8c	13	#include <lttng/msgpack.h>
21f58fb7	14	#include <lttng/event-notifier-notification.h>
437d5aa5	15	#include <lttng/events-internal.h>
11b589f6	16	#include <lttng/probe-user.h>
21f58fb7	17
99d223ad	18	/*
e699ee6c MD	19	* The capture buffer size needs to be below 1024 bytes to avoid the
	20	* frame to be larger than the 1024 limit enforced by the kernel. If we
	21	* ever need to increase it, we will need to use a memory allocation
	22	* scheme which allows allocating temporary memory chunks from the
	23	* instrumentation sites. This could be done by adapting lttng
	24	* tp-mempool to become nmi-safe and lock-free.
99d223ad FD	25	*/
	26	#define CAPTURE_BUFFER_SIZE 512
	27
f125ded4 MD	28	#define MSG_WRITE_NIL_LEN 1
f125ded4 MD	29
99d223ad FD	30	struct lttng_event_notifier_notification {
	31	int notification_fd;
	32	uint64_t event_notifier_token;
	33	uint8_t capture_buf[CAPTURE_BUFFER_SIZE];
	34	struct lttng_msgpack_writer writer;
	35	bool has_captures;
	36	};
	37
42f3ef8c FD	38	static
	39	int capture_enum(struct lttng_msgpack_writer *writer,
	40	struct lttng_interpreter_output *output)
	41	{
	42	int ret;
	43
	44	/*
	45	* Enums are captured as a map containing 2 key-value pairs. Such as:
	46	* - type: enum
	47	* value: 177
	48	*/
	49	ret = lttng_msgpack_begin_map(writer, 2);
	50	if (ret) {
42f3ef8c FD	51	goto end;
	52	}
	53
	54	ret = lttng_msgpack_write_str(writer, "type");
	55	if (ret) {
42f3ef8c FD	56	goto end;
	57	}
	58
	59	ret = lttng_msgpack_write_str(writer, "enum");
	60	if (ret) {
42f3ef8c FD	61	goto end;
	62	}
	63
	64	ret = lttng_msgpack_write_str(writer, "value");
	65	if (ret) {
42f3ef8c FD	66	goto end;
	67	}
	68
	69	switch (output->type) {
	70	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
	71	ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
	72	if (ret) {
42f3ef8c FD	73	goto end;
	74	}
	75	break;
	76	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
	77	ret = lttng_msgpack_write_signed_integer(writer, output->u.u);
	78	if (ret) {
42f3ef8c FD	79	goto end;
	80	}
	81	break;
	82	default:
736f2dd6	83	WARN_ON_ONCE(1);
60ae08f0 MD	84	ret = -1;
60ae08f0 MD	85	goto end;
42f3ef8c FD	86	}
	87
	88	ret = lttng_msgpack_end_map(writer);
42f3ef8c FD	89	end:
	90	return ret;
	91	}
	92
	93	static
60ae08f0 MD	94	int capture_sequence_element_signed(uint8_t *ptr,
	95	const struct lttng_kernel_type_integer *type,
	96	int64_t *_value)
42f3ef8c FD	97	{
	98	int64_t value = 0;
	99	unsigned int size = type->size;
11b589f6	100	bool user = type->user;
42f3ef8c FD	101	bool byte_order_reversed = type->reverse_byte_order;
	102
	103	switch (size) {
	104	case 8:
11b589f6 MD	105	{
	106	int8_t tmp;
	107
	108	if (user) {
	109	if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int8_t)))
60ae08f0	110	return -1;
11b589f6 MD	111	} else {
	112	tmp = *ptr;
	113	}
	114	value = tmp;
42f3ef8c	115	break;
11b589f6	116	}
42f3ef8c FD	117	case 16:
	118	{
	119	int16_t tmp;
11b589f6 MD	120
	121	if (user) {
	122	if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int16_t)))
60ae08f0	123	return -1;
11b589f6 MD	124	} else {
	125	tmp = (int16_t ) ptr;
	126	}
42f3ef8c FD	127	if (byte_order_reversed)
42f3ef8c FD	128	__swab16s(&tmp);
42f3ef8c FD	129	value = tmp;
	130	break;
	131	}
	132	case 32:
	133	{
	134	int32_t tmp;
11b589f6 MD	135
	136	if (user) {
	137	if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int32_t)))
60ae08f0	138	return -1;
11b589f6 MD	139	} else {
	140	tmp = (int32_t ) ptr;
	141	}
42f3ef8c FD	142	if (byte_order_reversed)
42f3ef8c FD	143	__swab32s(&tmp);
42f3ef8c FD	144	value = tmp;
	145	break;
	146	}
	147	case 64:
	148	{
	149	int64_t tmp;
11b589f6 MD	150
	151	if (user) {
	152	if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(int64_t)))
60ae08f0	153	return -1;
11b589f6 MD	154	} else {
	155	tmp = (int64_t ) ptr;
	156	}
42f3ef8c FD	157	if (byte_order_reversed)
42f3ef8c FD	158	__swab64s(&tmp);
42f3ef8c FD	159	value = tmp;
	160	break;
	161	}
	162	default:
736f2dd6	163	WARN_ON_ONCE(1);
60ae08f0	164	return -1;
42f3ef8c FD	165	}
42f3ef8c FD	166
60ae08f0 MD	167	*_value = value;
60ae08f0 MD	168	return 0;
42f3ef8c FD	169	}
	170
	171	static
60ae08f0 MD	172	int capture_sequence_element_unsigned(uint8_t *ptr,
	173	const struct lttng_kernel_type_integer *type,
	174	uint64_t *_value)
42f3ef8c FD	175	{
	176	uint64_t value = 0;
	177	unsigned int size = type->size;
11b589f6	178	bool user = type->user;
42f3ef8c FD	179	bool byte_order_reversed = type->reverse_byte_order;
	180
	181	switch (size) {
	182	case 8:
11b589f6 MD	183	{
	184	uint8_t tmp;
	185
	186	if (user) {
	187	if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint8_t)))
60ae08f0	188	return -1;
11b589f6 MD	189	} else {
	190	tmp = *ptr;
	191	}
	192	value = tmp;
42f3ef8c	193	break;
11b589f6	194	}
42f3ef8c FD	195	case 16:
	196	{
	197	uint16_t tmp;
11b589f6 MD	198
	199	if (user) {
	200	if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint16_t)))
60ae08f0	201	return -1;
11b589f6 MD	202	} else {
	203	tmp = (uint16_t ) ptr;
	204	}
42f3ef8c FD	205	if (byte_order_reversed)
42f3ef8c FD	206	__swab16s(&tmp);
42f3ef8c FD	207	value = tmp;
	208	break;
	209	}
	210	case 32:
	211	{
	212	uint32_t tmp;
11b589f6 MD	213
	214	if (user) {
	215	if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint32_t)))
60ae08f0	216	return -1;
11b589f6 MD	217	} else {
	218	tmp = (uint32_t ) ptr;
	219	}
42f3ef8c FD	220	if (byte_order_reversed)
42f3ef8c FD	221	__swab32s(&tmp);
42f3ef8c FD	222	value = tmp;
	223	break;
	224	}
	225	case 64:
	226	{
	227	uint64_t tmp;
11b589f6 MD	228
	229	if (user) {
	230	if (lttng_copy_from_user_check_nofault(&tmp, ptr, sizeof(uint64_t)))
60ae08f0	231	return -1;
11b589f6 MD	232	} else {
	233	tmp = (uint64_t ) ptr;
	234	}
42f3ef8c FD	235	if (byte_order_reversed)
42f3ef8c FD	236	__swab64s(&tmp);
42f3ef8c FD	237	value = tmp;
	238	break;
	239	}
	240	default:
736f2dd6	241	WARN_ON_ONCE(1);
60ae08f0	242	return -1;
42f3ef8c FD	243	}
42f3ef8c FD	244
60ae08f0 MD	245	*_value = value;
60ae08f0 MD	246	return 0;
42f3ef8c FD	247	}
42f3ef8c FD	248
11a1e5f2	249	static
42f3ef8c FD	250	int capture_sequence(struct lttng_msgpack_writer *writer,
	251	struct lttng_interpreter_output *output)
	252	{
437d5aa5 MD	253	const struct lttng_kernel_type_integer *integer_type = NULL;
437d5aa5 MD	254	const struct lttng_kernel_type_common *nested_type;
42f3ef8c FD	255	uint8_t *ptr;
	256	bool signedness;
	257	int ret, i;
	258
	259	ret = lttng_msgpack_begin_array(writer, output->u.sequence.nr_elem);
	260	if (ret) {
42f3ef8c FD	261	goto end;
	262	}
	263
	264	ptr = (uint8_t *) output->u.sequence.ptr;
	265	nested_type = output->u.sequence.nested_type;
12bb2edb MD	266	switch (nested_type->type) {
12bb2edb MD	267	case lttng_kernel_type_integer:
437d5aa5	268	integer_type = lttng_kernel_get_type_integer(nested_type);
42f3ef8c	269	break;
437d5aa5	270	case lttng_kernel_type_enum:
42f3ef8c	271	/* Treat enumeration as an integer. */
437d5aa5	272	integer_type = lttng_kernel_get_type_integer(lttng_kernel_get_type_enum(nested_type)->container_type);
42f3ef8c FD	273	break;
	274	default:
	275	/* Capture of array of non-integer are not supported. */
736f2dd6 MD	276	WARN_ON_ONCE(1);
	277	ret = -1;
	278	goto end;
42f3ef8c FD	279	}
	280	signedness = integer_type->signedness;
	281	for (i = 0; i < output->u.sequence.nr_elem; i++) {
	282	if (signedness) {
60ae08f0 MD	283	int64_t v;
	284
	285	ret = capture_sequence_element_signed(ptr, integer_type, &v);
	286	if (ret) {
	287	goto end;
	288	}
	289	ret = lttng_msgpack_write_signed_integer(writer, v);
42f3ef8c	290	if (ret) {
42f3ef8c FD	291	goto end;
	292	}
	293	} else {
60ae08f0 MD	294	uint64_t v;
	295
	296	ret = capture_sequence_element_unsigned(ptr, integer_type, &v);
	297	if (ret) {
	298	goto end;
	299	}
	300	ret = lttng_msgpack_write_unsigned_integer(writer, v);
42f3ef8c	301	if (ret) {
42f3ef8c FD	302	goto end;
	303	}
	304	}
	305
	306	/*
	307	* We assume that alignment is smaller or equal to the size.
	308	* This currently holds true but if it changes in the future,
	309	* we will want to change the pointer arithmetics below to
	310	* take into account that the next element might be further
	311	* away.
	312	*/
736f2dd6	313	WARN_ON_ONCE(integer_type->alignment > integer_type->size);
42f3ef8c FD	314
	315	/* Size is in number of bits. */
	316	ptr += (integer_type->size / CHAR_BIT) ;
	317	}
	318
	319	ret = lttng_msgpack_end_array(writer);
42f3ef8c FD	320	end:
	321	return ret;
	322	}
	323
99d223ad FD	324	static
	325	int notification_append_capture(
	326	struct lttng_event_notifier_notification *notif,
	327	struct lttng_interpreter_output *output)
	328	{
	329	struct lttng_msgpack_writer *writer = &notif->writer;
	330	int ret = 0;
	331
	332	switch (output->type) {
	333	case LTTNG_INTERPRETER_TYPE_S64:
	334	ret = lttng_msgpack_write_signed_integer(writer, output->u.s);
99d223ad FD	335	break;
	336	case LTTNG_INTERPRETER_TYPE_U64:
	337	ret = lttng_msgpack_write_unsigned_integer(writer, output->u.u);
99d223ad FD	338	break;
99d223ad FD	339	case LTTNG_INTERPRETER_TYPE_STRING:
8915cf5e MD	340	if (output->u.str.user) {
	341	ret = lttng_msgpack_write_user_str(writer, output->u.str.user_str);
	342	} else {
	343	ret = lttng_msgpack_write_str(writer, output->u.str.str);
	344	}
99d223ad FD	345	break;
	346	case LTTNG_INTERPRETER_TYPE_SEQUENCE:
	347	ret = capture_sequence(writer, output);
99d223ad FD	348	break;
	349	case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM:
	350	case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM:
	351	ret = capture_enum(writer, output);
99d223ad FD	352	break;
	353	default:
	354	ret = -1;
736f2dd6	355	WARN_ON_ONCE(1);
99d223ad	356	}
99d223ad FD	357	return ret;
	358	}
	359
	360	static
	361	int notification_append_empty_capture(
	362	struct lttng_event_notifier_notification *notif)
	363	{
736f2dd6	364	return lttng_msgpack_write_nil(&notif->writer);
99d223ad FD	365	}
	366
	367	static
	368	int notification_init(struct lttng_event_notifier_notification *notif,
a67ba386	369	struct lttng_kernel_event_notifier *event_notifier)
99d223ad FD	370	{
	371	struct lttng_msgpack_writer *writer = &notif->writer;
	372	int ret = 0;
	373
	374	notif->has_captures = false;
	375
a67ba386	376	if (event_notifier->priv->num_captures > 0) {
99d223ad FD	377	lttng_msgpack_writer_init(writer, notif->capture_buf,
	378	CAPTURE_BUFFER_SIZE);
	379
a67ba386	380	ret = lttng_msgpack_begin_array(writer, event_notifier->priv->num_captures);
99d223ad	381	if (ret) {
99d223ad FD	382	goto end;
	383	}
	384
	385	notif->has_captures = true;
	386	}
	387
	388	end:
	389	return ret;
	390	}
	391
99f52fcc	392	static
a67ba386	393	void record_error(struct lttng_kernel_event_notifier *event_notifier)
99f52fcc FD	394	{
99f52fcc FD	395
a67ba386	396	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
ab04d7b1	397	struct lttng_counter *error_counter;
99f52fcc FD	398	size_t dimension_index[1];
	399	int ret;
	400
ab04d7b1	401	/*
a27c8df7 MJ	402	* smp_load_acquire paired with smp_store_release orders creation of
	403	* the error counter and setting error_counter_len before the
	404	* error_counter is used.
ab04d7b1	405	*/
a27c8df7	406	error_counter = smp_load_acquire(&event_notifier_group->error_counter);
a83d6831	407	/* This group may not have an error counter attached to it. */
ab04d7b1	408	if (!error_counter)
a83d6831 FD	409	return;
a83d6831 FD	410
a67ba386	411	dimension_index[0] = event_notifier->priv->error_counter_index;
99f52fcc	412
ab04d7b1	413	ret = error_counter->ops->counter_add(error_counter->counter,
99f52fcc FD	414	dimension_index, 1);
	415	if (ret)
	416	WARN_ON_ONCE(1);
	417	}
	418
99d223ad FD	419	static
99d223ad FD	420	void notification_send(struct lttng_event_notifier_notification *notif,
a67ba386	421	struct lttng_kernel_event_notifier *event_notifier)
21f58fb7	422	{
a67ba386	423	struct lttng_event_notifier_group *event_notifier_group = event_notifier->priv->group;
8a57ec02	424	struct lttng_kernel_ring_buffer_ctx ctx;
606828e4	425	struct lttng_kernel_abi_event_notifier_notification kernel_notif;
99d223ad	426	size_t capture_buffer_content_len, reserve_size;
21f58fb7 FD	427	int ret;
21f58fb7 FD	428
99d223ad	429	reserve_size = sizeof(kernel_notif);
a67ba386	430	kernel_notif.token = event_notifier->priv->parent.user_token;
21f58fb7	431
99d223ad FD	432	if (notif->has_captures) {
	433	capture_buffer_content_len = notif->writer.write_pos - notif->writer.buffer;
	434	} else {
	435	capture_buffer_content_len = 0;
	436	}
	437
	438	WARN_ON_ONCE(capture_buffer_content_len > CAPTURE_BUFFER_SIZE);
	439
	440	reserve_size += capture_buffer_content_len;
	441	kernel_notif.capture_buf_size = capture_buffer_content_len;
	442
8a445457	443	lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, reserve_size,
b1199bd3	444	lttng_alignof(kernel_notif), NULL);
c2fb9c1c	445	ret = event_notifier_group->ops->event_reserve(&ctx);
21f58fb7	446	if (ret < 0) {
99f52fcc	447	record_error(event_notifier);
21f58fb7 FD	448	return;
21f58fb7 FD	449	}
99d223ad	450
99d223ad FD	451	/* Write the notif structure. */
99d223ad FD	452	event_notifier_group->ops->event_write(&ctx, &kernel_notif,
f5ffbd77	453	sizeof(kernel_notif), lttng_alignof(kernel_notif));
99d223ad FD	454
	455	/*
	456	* Write the capture buffer. No need to realigned as the below is a raw
	457	* char* buffer.
	458	*/
	459	event_notifier_group->ops->event_write(&ctx, &notif->capture_buf,
f5ffbd77	460	capture_buffer_content_len, 1);
99d223ad	461
21f58fb7 FD	462	event_notifier_group->ops->event_commit(&ctx);
	463	irq_work_queue(&event_notifier_group->wakeup_pending);
	464	}
99d223ad	465
f125ded4 MD	466	/*
	467	* Validate that the buffer has enough room to hold empty capture fields.
	468	*/
	469	static
	470	bool validate_buffer_len(struct lttng_event_notifier_notification *notif, size_t captures_left)
	471	{
	472	if (notif->writer.end_write_pos - notif->writer.write_pos < MSG_WRITE_NIL_LEN * captures_left)
	473	return false;
	474	return true;
	475	}
	476
a67ba386	477	void lttng_event_notifier_notification_send(struct lttng_kernel_event_notifier *event_notifier,
c3eddb2e	478	const char *stack_data,
a92e844e	479	struct lttng_kernel_probe_ctx *probe_ctx,
a67ba386	480	struct lttng_kernel_notification_ctx *notif_ctx)
99d223ad FD	481	{
99d223ad FD	482	struct lttng_event_notifier_notification notif = { 0 };
f125ded4	483	size_t captures_left;
99d223ad	484
f125ded4 MD	485	if (notification_init(&notif, event_notifier))
	486	goto error;
	487
	488	captures_left = event_notifier->priv->num_captures;
	489	if (!validate_buffer_len(&notif, captures_left))
	490	goto error;
99d223ad	491
c3eddb2e	492	if (unlikely(notif_ctx->eval_capture)) {
218585b9	493	struct lttng_kernel_bytecode_runtime *capture_bc_runtime;
99d223ad FD	494
	495	/*
	496	* Iterate over all the capture bytecodes. If the interpreter
	497	* functions returns successfully, append the value of the
	498	* `output` parameter to the capture buffer. If the interpreter
	499	* fails, append an empty capture to the buffer.
	500	*/
c3eddb2e	501	list_for_each_entry_rcu(capture_bc_runtime,
a67ba386	502	&event_notifier->priv->capture_bytecode_runtime_head, node) {
99d223ad	503	struct lttng_interpreter_output output;
736f2dd6	504	uint8_t *save_pos;
f125ded4	505	int ret = -1;
99d223ad	506
736f2dd6	507	lttng_msgpack_save_writer_pos(&notif.writer, &save_pos);
f125ded4	508	captures_left--;
8a445457 MD	509	if (capture_bc_runtime->interpreter_func(capture_bc_runtime,
8a445457 MD	510	stack_data, probe_ctx, &output) == LTTNG_KERNEL_BYTECODE_INTERPRETER_OK)
99d223ad	511	ret = notification_append_capture(&notif, &output);
f125ded4	512	if (ret \|\| !validate_buffer_len(&notif, captures_left)) {
736f2dd6	513	/*
f125ded4 MD	514	* On append capture error or if the generated
	515	* buffer data would not leave enough room to
	516	* write empty capture fields for the remaining
	517	* fields, skip the field capture by restoring
	518	* the msgpack writer position and writing an
	519	* empty capture field.
736f2dd6 MD	520	*/
736f2dd6 MD	521	lttng_msgpack_restore_writer_pos(&notif.writer, save_pos);
f125ded4 MD	522	ret = notification_append_empty_capture(&notif);
f125ded4 MD	523	WARN_ON_ONCE(ret);
736f2dd6	524	}
99d223ad FD	525	}
	526	}
	527
f125ded4 MD	528	if (notif.has_captures && lttng_msgpack_end_array(&notif.writer))
	529	goto error;
	530
99d223ad FD	531	/*
	532	* Send the notification (including the capture buffer) to the
	533	* sessiond.
	534	*/
	535	notification_send(&notif, event_notifier);
f125ded4 MD	536	return;
	537
	538	error:
	539	record_error(event_notifier);
99d223ad FD	540	return;
99d223ad FD	541	}