From 99d223adfae9b68800e91bda470c92acb14efae6 Mon Sep 17 00:00:00 2001 From: Francis Deslauriers Date: Thu, 2 Apr 2020 15:14:45 -0400 Subject: [PATCH] Implement capturing payload on event notifier Signed-off-by: Francis Deslauriers Signed-off-by: Mathieu Desnoyers Change-Id: Ibdaa07ecfe36ecd1c340942821503ea4cb494e2c --- include/lttng/abi.h | 18 ++ include/lttng/event-notifier-notification.h | 4 +- include/lttng/events.h | 21 +- include/lttng/lttng-bytecode.h | 10 + include/lttng/tracepoint-event-impl.h | 59 +++-- src/lttng-abi.c | 14 ++ src/lttng-bytecode-interpreter.c | 17 ++ src/lttng-bytecode.c | 22 +- src/lttng-event-notifier-notification.c | 202 ++++++++++++++++-- src/lttng-events.c | 59 +++++ src/lttng-ring-buffer-event-notifier-client.h | 40 +++- src/probes/lttng-kprobes.c | 2 +- src/probes/lttng-uprobes.c | 2 +- 13 files changed, 420 insertions(+), 50 deletions(-) diff --git a/include/lttng/abi.h b/include/lttng/abi.h index c7c6dc19..071133f6 100644 --- a/include/lttng/abi.h +++ b/include/lttng/abi.h @@ -144,6 +144,13 @@ struct lttng_kernel_event_notifier { char padding[LTTNG_KERNEL_EVENT_NOTIFIER_PADDING1]; } __attribute__((packed)); +#define LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING 32 +struct lttng_kernel_event_notifier_notification { + uint64_t token; + uint16_t capture_buf_size; + char padding[LTTNG_KERNEL_EVENT_NOTIFIER_NOTIFICATION_PADDING]; +} __attribute__((packed)); + struct lttng_kernel_tracer_version { uint32_t major; uint32_t minor; @@ -243,6 +250,14 @@ struct lttng_kernel_filter_bytecode { char data[0]; } __attribute__((packed)); +#define LTTNG_KERNEL_CAPTURE_BYTECODE_MAX_LEN 65536 +struct lttng_kernel_capture_bytecode { + uint32_t len; + uint32_t reloc_offset; + uint64_t seqnum; + char data[0]; +} __attribute__((packed)); + enum lttng_kernel_tracker_type { LTTNG_KERNEL_TRACKER_UNKNOWN = -1, @@ -337,6 +352,9 @@ struct lttng_kernel_tracker_args { #define LTTNG_KERNEL_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD \ _IO(0xF6, 0xB1) +/* Event notifier file descriptor ioctl */ +#define LTTNG_KERNEL_CAPTURE _IO(0xF6, 0xB8) + /* * LTTng-specific ioctls for the lib ringbuffer. * diff --git a/include/lttng/event-notifier-notification.h b/include/lttng/event-notifier-notification.h index f5ef9b98..b044e2fa 100644 --- a/include/lttng/event-notifier-notification.h +++ b/include/lttng/event-notifier-notification.h @@ -10,6 +10,8 @@ #include -void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_notifier); +void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_notifier, + struct lttng_probe_ctx *lttng_probe_ctx, + const char *stack_data); #endif /* _LTTNG_EVENT_NOTIFIER_NOTIFICATION_H */ diff --git a/include/lttng/events.h b/include/lttng/events.h index 3abdb805..80de5050 100644 --- a/include/lttng/events.h +++ b/include/lttng/events.h @@ -234,6 +234,7 @@ enum lttng_event_type { enum lttng_bytecode_node_type { LTTNG_BYTECODE_NODE_TYPE_FILTER, + LTTNG_BYTECODE_NODE_TYPE_CAPTURE, }; struct lttng_bytecode_node { @@ -257,6 +258,8 @@ enum lttng_bytecode_interpreter_ret { /* Other bits are kept for future use. */ }; +struct lttng_interpreter_output; + struct lttng_bytecode_runtime { /* Associated bytecode */ struct lttng_bytecode_node *bc; @@ -264,6 +267,10 @@ struct lttng_bytecode_runtime { uint64_t (*filter)(void *filter_data, struct lttng_probe_ctx *lttng_probe_ctx, const char *filter_stack_data); + uint64_t (*capture)(void *filter_data, + struct lttng_probe_ctx *lttng_probe_ctx, + const char *capture_stack_data, + struct lttng_interpreter_output *output); } interpreter_funcs; int link_failed; struct list_head node; /* list of bytecode runtime in event */ @@ -373,9 +380,13 @@ struct lttng_event_notifier { struct hlist_node hlist; /* session ht of event_notifiers */ /* list of struct lttng_bytecode_runtime, sorted by seqnum */ struct list_head filter_bytecode_runtime_head; + size_t num_captures; + struct list_head capture_bytecode_runtime_head; int has_enablers_without_bytecode; - void (*send_notification)(struct lttng_event_notifier *event_notifier); + void (*send_notification)(struct lttng_event_notifier *event_notifier, + struct lttng_probe_ctx *lttng_probe_ctx, + const char *interpreter_stack_data); struct lttng_event_notifier_group *group; /* Weak ref */ }; @@ -417,8 +428,13 @@ struct lttng_event_notifier_enabler { struct lttng_enabler base; struct list_head node; /* List of event_notifier enablers */ struct lttng_event_notifier_group *group; + + /* head list of struct lttng_bytecode_node */ + struct list_head capture_bytecode_head; + uint64_t num_captures; }; + static inline struct lttng_enabler *lttng_event_enabler_as_enabler( struct lttng_event_enabler *event_enabler) @@ -900,6 +916,9 @@ int lttng_event_enabler_attach_filter_bytecode(struct lttng_event_enabler *event int lttng_event_notifier_enabler_attach_filter_bytecode( struct lttng_event_notifier_enabler *event_notifier_enabler, struct lttng_kernel_filter_bytecode __user *bytecode); +int lttng_event_notifier_enabler_attach_capture_bytecode( + struct lttng_event_notifier_enabler *event_notifier_enabler, + struct lttng_kernel_capture_bytecode __user *bytecode); void lttng_enabler_link_bytecode(const struct lttng_event_desc *event_desc, struct lttng_ctx *ctx, diff --git a/include/lttng/lttng-bytecode.h b/include/lttng/lttng-bytecode.h index 3a11e230..ac4c738b 100644 --- a/include/lttng/lttng-bytecode.h +++ b/include/lttng/lttng-bytecode.h @@ -285,6 +285,7 @@ struct lttng_interpreter_output { const char *lttng_bytecode_print_op(enum bytecode_op op); void lttng_bytecode_filter_sync_state(struct lttng_bytecode_runtime *runtime); +void lttng_bytecode_capture_sync_state(struct lttng_bytecode_runtime *runtime); int lttng_bytecode_validate(struct bytecode_runtime *bytecode); int lttng_bytecode_specialize(const struct lttng_event_desc *event_desc, struct bytecode_runtime *bytecode); @@ -296,4 +297,13 @@ uint64_t lttng_bytecode_filter_interpret(void *filter_data, struct lttng_probe_ctx *lttng_probe_ctx, const char *filter_stack_data); +uint64_t lttng_bytecode_capture_interpret_false(void *capture_data, + struct lttng_probe_ctx *lttng_probe_ctx, + const char *capture_stack_data, + struct lttng_interpreter_output *output); +uint64_t lttng_bytecode_capture_interpret(void *capture_data, + struct lttng_probe_ctx *lttng_probe_ctx, + const char *capture_stack_data, + struct lttng_interpreter_output *output); + #endif /* _LTTNG_FILTER_H */ diff --git a/include/lttng/tracepoint-event-impl.h b/include/lttng/tracepoint-event-impl.h index d8ad2163..079d7d37 100644 --- a/include/lttng/tracepoint-event-impl.h +++ b/include/lttng/tracepoint-event-impl.h @@ -822,7 +822,7 @@ error: \ #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ static inline \ -void __event_prepare_filter_stack__##_name(char *__stack_data, \ +void __event_prepare_interpreter_stack__##_name(char *__stack_data, \ void *__tp_locvar) \ { \ struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar; \ @@ -833,7 +833,7 @@ void __event_prepare_filter_stack__##_name(char *__stack_data, \ #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ static inline \ -void __event_prepare_filter_stack__##_name(char *__stack_data, \ +void __event_prepare_interpreter_stack__##_name(char *__stack_data, \ void *__tp_locvar, _proto) \ { \ struct { _locvar } *tp_locvar __attribute__((unused)) = __tp_locvar; \ @@ -1246,7 +1246,7 @@ static void __event_probe__##_name(void *__data, _proto) \ struct lttng_bytecode_runtime *bc_runtime; \ int __filter_record = __event->has_enablers_without_bytecode; \ \ - __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \ + __event_prepare_interpreter_stack__##_name(__stackvar.__filter_stack_data, \ tp_locvar, _args); \ lttng_list_for_each_entry_rcu(bc_runtime, &__event->filter_bytecode_runtime_head, node) { \ if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ @@ -1342,7 +1342,7 @@ static void __event_probe__##_name(void *__data) \ struct lttng_bytecode_runtime *bc_runtime; \ int __filter_record = __event->has_enablers_without_bytecode; \ \ - __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \ + __event_prepare_interpreter_stack__##_name(__stackvar.__filter_stack_data, \ tp_locvar); \ lttng_list_for_each_entry_rcu(bc_runtime, &__event->filter_bytecode_runtime_head, node) { \ if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ @@ -1425,7 +1425,7 @@ static void __event_notifier_probe__##_name(void *__data, _proto) \ }; \ union { \ size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ - char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ + char __interpreter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ } __stackvar; \ struct probe_local_vars __tp_locvar; \ struct probe_local_vars *tp_locvar __attribute__((unused)) = \ @@ -1435,21 +1435,29 @@ static void __event_notifier_probe__##_name(void *__data, _proto) \ return; \ _code_pre \ if (unlikely(!list_empty(&__event_notifier->filter_bytecode_runtime_head))) { \ - struct lttng_bytecode_runtime *bc_runtime; \ + struct lttng_bytecode_runtime *bc_runtime; \ int __filter_record = __event_notifier->has_enablers_without_bytecode; \ - \ - __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \ - tp_locvar, _args); \ + \ + __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ + tp_locvar, _args); \ lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->filter_bytecode_runtime_head, node) { \ - if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ - __stackvar.__filter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ + if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ + __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ __filter_record = 1; \ } \ if (likely(!__filter_record)) \ goto __post; \ } \ \ - __event_notifier->send_notification(__event_notifier); \ + if (unlikely(!list_empty(&__event_notifier->capture_bytecode_runtime_head))) \ + __event_prepare_interpreter_stack__##_name( \ + __stackvar.__interpreter_stack_data, \ + tp_locvar, _args); \ + \ + __event_notifier->send_notification(__event_notifier, \ + &__lttng_probe_ctx, \ + __stackvar.__interpreter_stack_data); \ + \ __post: \ _code_post \ return; \ @@ -1468,7 +1476,7 @@ static void __event_notifier_probe__##_name(void *__data) \ }; \ union { \ size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ - char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ + char __interpreter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ } __stackvar; \ struct probe_local_vars __tp_locvar; \ struct probe_local_vars *tp_locvar __attribute__((unused)) = \ @@ -1477,22 +1485,29 @@ static void __event_notifier_probe__##_name(void *__data) \ if (unlikely(!READ_ONCE(__event_notifier->enabled))) \ return; \ _code_pre \ - if (unlikely(!list_empty(&__event_notifier->filter_bytecode_runtime_head))) { \ - struct lttng_bytecode_runtime *bc_runtime; \ - int __filter_record = __event_notifier->has_enablers_without_bytecode; \ - \ - __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \ - tp_locvar); \ + if (unlikely(!list_empty(&__event_notifier->filter_bytecode_runtime_head))) { \ + struct lttng_bytecode_runtime *bc_runtime; \ + int __filter_record = __event_notifier->has_enablers_without_bytecode; \ + \ + __event_prepare_interpreter_stack__##_name(__stackvar.__interpreter_stack_data, \ + tp_locvar); \ lttng_list_for_each_entry_rcu(bc_runtime, &__event_notifier->filter_bytecode_runtime_head, node) { \ - if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ - __stackvar.__filter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ + if (unlikely(bc_runtime->interpreter_funcs.filter(bc_runtime, &__lttng_probe_ctx, \ + __stackvar.__interpreter_stack_data) & LTTNG_INTERPRETER_RECORD_FLAG)) \ __filter_record = 1; \ } \ if (likely(!__filter_record)) \ goto __post; \ } \ \ - __event_notifier->send_notification(__event_notifier); \ + if (unlikely(!list_empty(&__event_notifier->capture_bytecode_runtime_head))) \ + __event_prepare_interpreter_stack__##_name( \ + __stackvar.__interpreter_stack_data, \ + tp_locvar); \ + \ + __event_notifier->send_notification(__event_notifier, \ + &__lttng_probe_ctx, \ + __stackvar.__interpreter_stack_data); \ __post: \ _code_post \ return; \ diff --git a/src/lttng-abi.c b/src/lttng-abi.c index be8bfe1e..344f18c2 100644 --- a/src/lttng-abi.c +++ b/src/lttng-abi.c @@ -1755,6 +1755,20 @@ long lttng_event_notifier_ioctl(struct file *file, unsigned int cmd, unsigned lo WARN_ON_ONCE(1); return -ENOSYS; } + + case LTTNG_KERNEL_CAPTURE: + switch (*evtype) { + case LTTNG_TYPE_EVENT: + return -EINVAL; + case LTTNG_TYPE_ENABLER: + event_notifier_enabler = file->private_data; + return lttng_event_notifier_enabler_attach_capture_bytecode( + event_notifier_enabler, + (struct lttng_kernel_capture_bytecode __user *) arg); + default: + WARN_ON_ONCE(1); + return -ENOSYS; + } case LTTNG_KERNEL_ADD_CALLSITE: switch (*evtype) { case LTTNG_TYPE_EVENT: diff --git a/src/lttng-bytecode-interpreter.c b/src/lttng-bytecode-interpreter.c index 8252cdf1..e183e330 100644 --- a/src/lttng-bytecode-interpreter.c +++ b/src/lttng-bytecode-interpreter.c @@ -214,6 +214,14 @@ uint64_t lttng_bytecode_filter_interpret_false(void *filter_data, return LTTNG_INTERPRETER_DISCARD; } +uint64_t lttng_bytecode_capture_interpret_false(void *filter_data, + struct lttng_probe_ctx *lttng_probe_ctx, + const char *capture_stack_data, + struct lttng_interpreter_output *output) +{ + return LTTNG_INTERPRETER_DISCARD; +} + #ifdef INTERPRETER_USE_SWITCH /* @@ -1782,6 +1790,15 @@ uint64_t lttng_bytecode_filter_interpret(void *filter_data, filter_stack_data, NULL); } +uint64_t lttng_bytecode_capture_interpret(void *capture_data, + struct lttng_probe_ctx *lttng_probe_ctx, + const char *capture_stack_data, + struct lttng_interpreter_output *output) +{ + return bytecode_interpret(capture_data, lttng_probe_ctx, + capture_stack_data, output); +} + #undef START_OP #undef OP #undef PO diff --git a/src/lttng-bytecode.c b/src/lttng-bytecode.c index 5b6cb117..5702929b 100644 --- a/src/lttng-bytecode.c +++ b/src/lttng-bytecode.c @@ -457,6 +457,9 @@ int link_bytecode(const struct lttng_event_desc *event_desc, case LTTNG_BYTECODE_NODE_TYPE_FILTER: runtime->p.interpreter_funcs.filter = lttng_bytecode_filter_interpret; break; + case LTTNG_BYTECODE_NODE_TYPE_CAPTURE: + runtime->p.interpreter_funcs.capture = lttng_bytecode_capture_interpret_false; + break; default: WARN_ON(1); } @@ -472,6 +475,9 @@ link_error: case LTTNG_BYTECODE_NODE_TYPE_FILTER: runtime->p.interpreter_funcs.filter = lttng_bytecode_filter_interpret_false; break; + case LTTNG_BYTECODE_NODE_TYPE_CAPTURE: + runtime->p.interpreter_funcs.capture = lttng_bytecode_capture_interpret_false; + break; default: WARN_ON(1); } @@ -492,10 +498,20 @@ void lttng_bytecode_filter_sync_state(struct lttng_bytecode_runtime *runtime) runtime->interpreter_funcs.filter = lttng_bytecode_filter_interpret; } +void lttng_bytecode_capture_sync_state(struct lttng_bytecode_runtime *runtime) +{ + struct lttng_bytecode_node *bc = runtime->bc; + + if (!bc->enabler->enabled || runtime->link_failed) + runtime->interpreter_funcs.capture = lttng_bytecode_capture_interpret_false; + else + runtime->interpreter_funcs.capture = lttng_bytecode_capture_interpret; +} + /* - * Given the lists of bytecode programs of an instance (trigger or event) and - * of a matching enabler, try to link all the enabler's bytecode programs with - * the instance. + * Given the lists of bytecode programs of an instance (event or event + * notifier) and of a matching enabler, try to link all the enabler's bytecode + * programs with the instance. * * This function is called after we confirmed that name enabler and the * instance are matching names (or glob pattern matching). diff --git a/src/lttng-event-notifier-notification.c b/src/lttng-event-notifier-notification.c index f0e833ad..f675de4b 100644 --- a/src/lttng-event-notifier-notification.c +++ b/src/lttng-event-notifier-notification.c @@ -12,9 +12,20 @@ #include #include -static -int capture_enum(struct lttng_msgpack_writer *writer, - struct lttng_interpreter_output *output) __attribute__ ((unused)); +/* + * FIXME: this probably too low but it needs to be below 1024 bytes to avoid + * the frame to be larger than the 1024 limit enforced by the kernel. + */ +#define CAPTURE_BUFFER_SIZE 512 + +struct lttng_event_notifier_notification { + int notification_fd; + uint64_t event_notifier_token; + uint8_t capture_buf[CAPTURE_BUFFER_SIZE]; + struct lttng_msgpack_writer writer; + bool has_captures; +}; + static int capture_enum(struct lttng_msgpack_writer *writer, struct lttng_interpreter_output *output) @@ -175,9 +186,6 @@ uint64_t capture_sequence_element_unsigned(uint8_t *ptr, return value; } -static -int capture_sequence(struct lttng_msgpack_writer *writer, - struct lttng_interpreter_output *output) __attribute__ ((unused)); int capture_sequence(struct lttng_msgpack_writer *writer, struct lttng_interpreter_output *output) { @@ -245,18 +253,122 @@ end: return ret; } -void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_notifier) +static +int notification_append_capture( + struct lttng_event_notifier_notification *notif, + struct lttng_interpreter_output *output) +{ + struct lttng_msgpack_writer *writer = ¬if->writer; + int ret = 0; + + switch (output->type) { + case LTTNG_INTERPRETER_TYPE_S64: + ret = lttng_msgpack_write_signed_integer(writer, output->u.s); + if (ret) { + WARN_ON_ONCE(1); + goto end; + } + break; + case LTTNG_INTERPRETER_TYPE_U64: + ret = lttng_msgpack_write_unsigned_integer(writer, output->u.u); + if (ret) { + WARN_ON_ONCE(1); + goto end; + } + break; + case LTTNG_INTERPRETER_TYPE_STRING: + ret = lttng_msgpack_write_str(writer, output->u.str.str); + if (ret) { + WARN_ON_ONCE(1); + goto end; + } + break; + case LTTNG_INTERPRETER_TYPE_SEQUENCE: + ret = capture_sequence(writer, output); + if (ret) { + WARN_ON_ONCE(1); + goto end; + } + break; + case LTTNG_INTERPRETER_TYPE_SIGNED_ENUM: + case LTTNG_INTERPRETER_TYPE_UNSIGNED_ENUM: + ret = capture_enum(writer, output); + if (ret) { + WARN_ON_ONCE(1); + goto end; + } + break; + default: + ret = -1; + WARN_ON(1); + } +end: + return ret; +} + +static +int notification_append_empty_capture( + struct lttng_event_notifier_notification *notif) +{ + int ret = lttng_msgpack_write_nil(¬if->writer); + if (ret) + WARN_ON_ONCE(1); + + return ret; +} + +static +int notification_init(struct lttng_event_notifier_notification *notif, + struct lttng_event_notifier *event_notifier) +{ + struct lttng_msgpack_writer *writer = ¬if->writer; + int ret = 0; + + notif->has_captures = false; + + if (event_notifier->num_captures > 0) { + lttng_msgpack_writer_init(writer, notif->capture_buf, + CAPTURE_BUFFER_SIZE); + + ret = lttng_msgpack_begin_array(writer, event_notifier->num_captures); + if (ret) { + WARN_ON_ONCE(1); + goto end; + } + + notif->has_captures = true; + } + +end: + return ret; +} + +static +void notification_send(struct lttng_event_notifier_notification *notif, + struct lttng_event_notifier *event_notifier) { struct lttng_event_notifier_group *event_notifier_group = event_notifier->group; struct lib_ring_buffer_ctx ctx; + struct lttng_kernel_event_notifier_notification kernel_notif; + size_t capture_buffer_content_len, reserve_size; int ret; - if (unlikely(!READ_ONCE(event_notifier->enabled))) - return; + reserve_size = sizeof(kernel_notif); + kernel_notif.token = event_notifier->user_token; - lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, NULL, - sizeof(event_notifier->user_token), - lttng_alignof(event_notifier->user_token), -1); + if (notif->has_captures) { + capture_buffer_content_len = notif->writer.write_pos - notif->writer.buffer; + } else { + capture_buffer_content_len = 0; + } + + WARN_ON_ONCE(capture_buffer_content_len > CAPTURE_BUFFER_SIZE); + + reserve_size += capture_buffer_content_len; + kernel_notif.capture_buf_size = capture_buffer_content_len; + + lib_ring_buffer_ctx_init(&ctx, event_notifier_group->chan, NULL, reserve_size, + lttng_alignof(kernel_notif), -1); ret = event_notifier_group->ops->event_reserve(&ctx, 0); if (ret < 0) { //TODO: error handling with counter maps @@ -264,9 +376,69 @@ void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_n WARN_ON_ONCE(1); return; } - lib_ring_buffer_align_ctx(&ctx, lttng_alignof(event_notifier->user_token)); - event_notifier_group->ops->event_write(&ctx, &event_notifier->user_token, - sizeof(event_notifier->user_token)); + + lib_ring_buffer_align_ctx(&ctx, lttng_alignof(kernel_notif)); + + /* Write the notif structure. */ + event_notifier_group->ops->event_write(&ctx, &kernel_notif, + sizeof(kernel_notif)); + + /* + * Write the capture buffer. No need to realigned as the below is a raw + * char* buffer. + */ + event_notifier_group->ops->event_write(&ctx, ¬if->capture_buf, + capture_buffer_content_len); + event_notifier_group->ops->event_commit(&ctx); irq_work_queue(&event_notifier_group->wakeup_pending); } + +void lttng_event_notifier_notification_send(struct lttng_event_notifier *event_notifier, + struct lttng_probe_ctx *lttng_probe_ctx, + const char *stack_data) +{ + struct lttng_event_notifier_notification notif = { 0 }; + int ret; + + if (unlikely(!READ_ONCE(event_notifier->enabled))) + return; + + ret = notification_init(¬if, event_notifier); + if (ret) { + WARN_ON_ONCE(1); + goto end; + } + + if (unlikely(!list_empty(&event_notifier->capture_bytecode_runtime_head))) { + struct lttng_bytecode_runtime *capture_bc_runtime; + + /* + * Iterate over all the capture bytecodes. If the interpreter + * functions returns successfully, append the value of the + * `output` parameter to the capture buffer. If the interpreter + * fails, append an empty capture to the buffer. + */ + list_for_each_entry(capture_bc_runtime, + &event_notifier->capture_bytecode_runtime_head, node) { + struct lttng_interpreter_output output; + + if (capture_bc_runtime->interpreter_funcs.capture(capture_bc_runtime, + lttng_probe_ctx, stack_data, &output) & LTTNG_INTERPRETER_RECORD_FLAG) + ret = notification_append_capture(¬if, &output); + else + ret = notification_append_empty_capture(¬if); + + if (ret) + printk(KERN_WARNING "Error appending capture to notification"); + } + } + + /* + * Send the notification (including the capture buffer) to the + * sessiond. + */ + notification_send(¬if, event_notifier); +end: + return; +} diff --git a/src/lttng-events.c b/src/lttng-events.c index 9b78ac56..75a19304 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -1048,11 +1048,13 @@ struct lttng_event_notifier *_lttng_event_notifier_create( event_notifier->group = event_notifier_group; event_notifier->user_token = token; + event_notifier->num_captures = 0; event_notifier->filter = filter; event_notifier->instrumentation = itype; event_notifier->evtype = LTTNG_TYPE_EVENT; event_notifier->send_notification = lttng_event_notifier_notification_send; INIT_LIST_HEAD(&event_notifier->filter_bytecode_runtime_head); + INIT_LIST_HEAD(&event_notifier->capture_bytecode_runtime_head); INIT_LIST_HEAD(&event_notifier->enablers_ref_head); switch (itype) { @@ -2120,6 +2122,13 @@ int lttng_event_notifier_enabler_ref_event_notifiers( lttng_enabler_link_bytecode(event_notifier->desc, lttng_static_ctx, &event_notifier->filter_bytecode_runtime_head, <tng_event_notifier_enabler_as_enabler(event_notifier_enabler)->filter_bytecode_head); + + /* Link capture bytecodes if not linked yet. */ + lttng_enabler_link_bytecode(event_notifier->desc, + lttng_static_ctx, &event_notifier->capture_bytecode_runtime_head, + &event_notifier_enabler->capture_bytecode_head); + + event_notifier->num_captures = event_notifier_enabler->num_captures; } return 0; } @@ -2318,6 +2327,9 @@ struct lttng_event_notifier_enabler *lttng_event_notifier_enabler_create( event_notifier_enabler->base.format_type = format_type; INIT_LIST_HEAD(&event_notifier_enabler->base.filter_bytecode_head); + INIT_LIST_HEAD(&event_notifier_enabler->capture_bytecode_head); + + event_notifier_enabler->num_captures = 0; memcpy(&event_notifier_enabler->base.event_param, &event_notifier_param->event, sizeof(event_notifier_enabler->base.event_param)); @@ -2375,6 +2387,48 @@ error: return ret; } +int lttng_event_notifier_enabler_attach_capture_bytecode( + struct lttng_event_notifier_enabler *event_notifier_enabler, + struct lttng_kernel_capture_bytecode __user *bytecode) +{ + struct lttng_bytecode_node *bytecode_node; + struct lttng_enabler *enabler = + lttng_event_notifier_enabler_as_enabler(event_notifier_enabler); + uint32_t bytecode_len; + int ret; + + ret = get_user(bytecode_len, &bytecode->len); + if (ret) + return ret; + + bytecode_node = lttng_kvzalloc(sizeof(*bytecode_node) + bytecode_len, + GFP_KERNEL); + if (!bytecode_node) + return -ENOMEM; + + ret = copy_from_user(&bytecode_node->bc, bytecode, + sizeof(*bytecode) + bytecode_len); + if (ret) + goto error_free; + + bytecode_node->type = LTTNG_BYTECODE_NODE_TYPE_CAPTURE; + bytecode_node->enabler = enabler; + + /* Enforce length based on allocated size */ + bytecode_node->bc.len = bytecode_len; + list_add_tail(&bytecode_node->node, &event_notifier_enabler->capture_bytecode_head); + + event_notifier_enabler->num_captures++; + + lttng_event_notifier_group_sync_enablers(event_notifier_enabler->group); + goto end; + +error_free: + lttng_kvfree(bytecode_node); +end: + return ret; +} + int lttng_event_notifier_add_callsite(struct lttng_event_notifier *event_notifier, struct lttng_kernel_event_callsite __user *callsite) { @@ -2565,6 +2619,11 @@ void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier_group list_for_each_entry(runtime, &event_notifier->filter_bytecode_runtime_head, node) lttng_bytecode_filter_sync_state(runtime); + + /* Enable captures */ + list_for_each_entry(runtime, + &event_notifier->capture_bytecode_runtime_head, node) + lttng_bytecode_capture_sync_state(runtime); } } diff --git a/src/lttng-ring-buffer-event-notifier-client.h b/src/lttng-ring-buffer-event-notifier-client.h index ea0ab66e..a857c8f7 100644 --- a/src/lttng-ring-buffer-event-notifier-client.h +++ b/src/lttng-ring-buffer-event-notifier-client.h @@ -7,9 +7,11 @@ * Copyright (C) 2010-2020 Mathieu Desnoyers */ +#include #include #include #include /* for wrapper_vmalloc_sync_mappings() */ +#include #include #include @@ -20,6 +22,7 @@ struct event_notifier_packet_header { }; struct event_notifier_record_header { + uint32_t payload_len; /* in bytes */ uint8_t header_end[0]; /* End of header */ }; @@ -38,7 +41,17 @@ size_t record_header_size(const struct lib_ring_buffer_config *config, struct lib_ring_buffer_ctx *ctx, void *client_ctx) { - return 0; + size_t orig_offset = offset; + size_t padding; + + padding = lib_ring_buffer_align(offset, lttng_alignof(uint32_t)); + offset += padding; + + offset += sizeof(uint32_t); + + *pre_header_padding = padding; + + return offset - orig_offset; } #include @@ -55,7 +68,8 @@ size_t client_record_header_size(const struct lib_ring_buffer_config *config, struct lib_ring_buffer_ctx *ctx, void *client_ctx) { - return 0; + return record_header_size(config, chan, offset, + pre_header_padding, ctx, client_ctx); } /** @@ -169,10 +183,7 @@ static void client_record_get(const struct lib_ring_buffer_config *config, offsetof(struct event_notifier_record_header, header_end)); CHAN_WARN_ON(chan, ret != offsetof(struct event_notifier_record_header, header_end)); *header_len = offsetof(struct event_notifier_record_header, header_end); - /* - * Currently, only 64-bit event notifier ID. - */ - *payload_len = sizeof(uint64_t); + *payload_len = header.payload_len; *timestamp = 0; } @@ -273,6 +284,21 @@ void lttng_buffer_read_close(struct lib_ring_buffer *buf) lib_ring_buffer_release_read(buf); } +static +void lttng_write_event_notifier_header(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx) +{ + uint32_t data_size; + + WARN_ON_ONCE(ctx->data_size > U32_MAX); + + data_size = (uint32_t) ctx->data_size; + + lib_ring_buffer_write(config, ctx, &data_size, sizeof(data_size)); + + lib_ring_buffer_align_ctx(ctx, ctx->largest_align); +} + static int lttng_event_reserve(struct lib_ring_buffer_ctx *ctx, uint32_t event_id) { @@ -283,6 +309,8 @@ int lttng_event_reserve(struct lib_ring_buffer_ctx *ctx, uint32_t event_id) return ret; lib_ring_buffer_backend_get_pages(&client_config, ctx, &ctx->backend_pages); + + lttng_write_event_notifier_header(&client_config, ctx); return 0; } diff --git a/src/probes/lttng-kprobes.c b/src/probes/lttng-kprobes.c index 4e80f1ef..6824088c 100644 --- a/src/probes/lttng-kprobes.c +++ b/src/probes/lttng-kprobes.c @@ -58,7 +58,7 @@ int lttng_kprobes_event_notifier_handler_pre(struct kprobe *p, struct pt_regs *r if (unlikely(!READ_ONCE(event_notifier->enabled))) return 0; - event_notifier->send_notification(event_notifier); + event_notifier->send_notification(event_notifier, NULL, NULL); return 0; } diff --git a/src/probes/lttng-uprobes.c b/src/probes/lttng-uprobes.c index 368828e3..b9007146 100644 --- a/src/probes/lttng-uprobes.c +++ b/src/probes/lttng-uprobes.c @@ -73,7 +73,7 @@ int lttng_uprobes_event_notifier_handler_pre(struct uprobe_consumer *uc, struct if (unlikely(!READ_ONCE(event_notifier->enabled))) return 0; - event_notifier->send_notification(event_notifier); + event_notifier->send_notification(event_notifier, NULL, NULL); return 0; } -- 2.34.1