From 114667d5aa4ec11334787e04a9f90fe9184fc16f Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 17 Mar 2016 18:09:32 -0400 Subject: [PATCH] Implement shadow stack for dynamic len Signed-off-by: Mathieu Desnoyers --- lttng-events.c | 3 + lttng-events.h | 10 ++++ lttng-probes.c | 13 +++++ probes/lttng-tracepoint-event-impl.h | 83 +++++++++++++++++++--------- 4 files changed, 84 insertions(+), 25 deletions(-) diff --git a/lttng-events.c b/lttng-events.c index b748cce6..a5fe3e86 100644 --- a/lttng-events.c +++ b/lttng-events.c @@ -2444,6 +2444,9 @@ static int __init lttng_events_init(void) if (ret) return ret; ret = wrapper_get_pageblock_flags_mask_init(); + if (ret) + return ret; + ret = lttng_probes_init(); if (ret) return ret; ret = lttng_context_init(); diff --git a/lttng-events.h b/lttng-events.h index fe762847..56f5d3ac 100644 --- a/lttng-events.h +++ b/lttng-events.h @@ -447,6 +447,14 @@ struct lttng_metadata_stream { uint64_t version; /* Current version of the metadata cache */ }; +#define LTTNG_DYNAMIC_LEN_STACK_SIZE 128 + +struct lttng_dynamic_len_stack { + size_t stack[LTTNG_DYNAMIC_LEN_STACK_SIZE]; + size_t offset; +}; + +DECLARE_PER_CPU(struct lttng_dynamic_len_stack, lttng_dynamic_len_stack); /* * struct lttng_pid_tracker declared in header due to deferencing of *v @@ -628,6 +636,8 @@ int lttng_enabler_attach_bytecode(struct lttng_enabler *enabler, void lttng_enabler_event_link_bytecode(struct lttng_event *event, struct lttng_enabler *enabler); +int lttng_probes_init(void); + extern struct lttng_ctx *lttng_static_ctx; int lttng_context_init(void); diff --git a/lttng-probes.c b/lttng-probes.c index dd2b2ce7..51b0f865 100644 --- a/lttng-probes.c +++ b/lttng-probes.c @@ -44,6 +44,10 @@ static LIST_HEAD(lazy_probe_init); */ static int lazy_nesting; +DEFINE_PER_CPU(struct lttng_dynamic_len_stack, lttng_dynamic_len_stack); + +EXPORT_PER_CPU_SYMBOL_GPL(lttng_dynamic_len_stack); + /* * Called under sessions lock. */ @@ -322,3 +326,12 @@ const struct file_operations lttng_tracepoint_list_fops = { .llseek = seq_lseek, .release = seq_release, }; + +int lttng_probes_init(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + per_cpu_ptr(<tng_dynamic_len_stack, cpu)->offset = 0; + return 0; +} diff --git a/probes/lttng-tracepoint-event-impl.h b/probes/lttng-tracepoint-event-impl.h index ba906ac4..42a4be30 100644 --- a/probes/lttng-tracepoint-event-impl.h +++ b/probes/lttng-tracepoint-event-impl.h @@ -370,9 +370,15 @@ static void __event_probe__##_name(void *__data); __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_length_type)); \ __event_len += sizeof(_length_type); \ __event_len += lib_ring_buffer_align(__event_len, lttng_alignof(_type)); \ - __dynamic_len[__dynamic_len_idx] = (_src_length); \ - __event_len += sizeof(_type) * __dynamic_len[__dynamic_len_idx]; \ - __dynamic_len_idx++; + { \ + size_t __seqlen = (_src_length); \ + \ + if (unlikely(++this_cpu_ptr(<tng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \ + goto error; \ + barrier(); /* reserve before use. */ \ + this_cpu_ptr(<tng_dynamic_len_stack)->stack[this_cpu_ptr(<tng_dynamic_len_stack)->offset - 1] = __seqlen; \ + __event_len += sizeof(_type) * __seqlen; \ + } #undef _ctf_sequence_bitfield #define _ctf_sequence_bitfield(_type, _item, _src, \ @@ -387,12 +393,16 @@ static void __event_probe__##_name(void *__data); */ #undef _ctf_string #define _ctf_string(_item, _src, _user, _nowrite) \ - if (_user) \ - __event_len += __dynamic_len[__dynamic_len_idx++] = \ + if (unlikely(++this_cpu_ptr(<tng_dynamic_len_stack)->offset >= LTTNG_DYNAMIC_LEN_STACK_SIZE)) \ + goto error; \ + barrier(); /* reserve before use. */ \ + if (_user) { \ + __event_len += this_cpu_ptr(<tng_dynamic_len_stack)->stack[this_cpu_ptr(<tng_dynamic_len_stack)->offset - 1] = \ max_t(size_t, lttng_strlen_user_inatomic(_src), 1); \ - else \ - __event_len += __dynamic_len[__dynamic_len_idx++] = \ - strlen(_src) + 1; + } else { \ + __event_len += this_cpu_ptr(<tng_dynamic_len_stack)->stack[this_cpu_ptr(<tng_dynamic_len_stack)->offset - 1] = \ + strlen(_src) + 1; \ + } #undef TP_PROTO #define TP_PROTO(...) __VA_ARGS__ @@ -405,8 +415,7 @@ static void __event_probe__##_name(void *__data); #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE(_name, _proto, _args, _locvar, _code_pre, _fields, _code_post) \ -static inline size_t __event_get_size__##_name(size_t *__dynamic_len, \ - void *__tp_locvar, _proto) \ +static inline ssize_t __event_get_size__##_name(void *__tp_locvar, _proto) \ { \ size_t __event_len = 0; \ unsigned int __dynamic_len_idx __attribute__((unused)) = 0; \ @@ -414,12 +423,15 @@ static inline size_t __event_get_size__##_name(size_t *__dynamic_len, \ \ _fields \ return __event_len; \ + \ +error: \ + __attribute__((unused)); \ + return -1; \ } #undef LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS #define LTTNG_TRACEPOINT_EVENT_CLASS_CODE_NOARGS(_name, _locvar, _code_pre, _fields, _code_post) \ -static inline size_t __event_get_size__##_name(size_t *__dynamic_len, \ - void *__tp_locvar) \ +static inline ssize_t __event_get_size__##_name(void *__tp_locvar) \ { \ size_t __event_len = 0; \ unsigned int __dynamic_len_idx __attribute__((unused)) = 0; \ @@ -427,6 +439,10 @@ static inline size_t __event_get_size__##_name(size_t *__dynamic_len, \ \ _fields \ return __event_len; \ + \ +error: \ + __attribute__((unused)); \ + return -1; \ } #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) @@ -791,7 +807,7 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar) \ #define _ctf_sequence_encoded(_type, _item, _src, _length_type, \ _src_length, _encoding, _byte_order, _base, _user, _nowrite) \ { \ - _length_type __tmpl = __stackvar.__dynamic_len[__dynamic_len_idx]; \ + _length_type __tmpl = this_cpu_ptr(<tng_dynamic_len_stack)->stack[__dynamic_len_idx]; \ lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\ __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\ } \ @@ -810,7 +826,7 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar) \ _length_type, _src_length, \ _user, _nowrite) \ { \ - _length_type __tmpl = __stackvar.__dynamic_len[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \ + _length_type __tmpl = this_cpu_ptr(<tng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \ lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\ __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\ } \ @@ -831,7 +847,7 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar) \ _length_type, _src_length, \ _user, _nowrite) \ { \ - _length_type __tmpl = __stackvar.__dynamic_len[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \ + _length_type __tmpl = this_cpu_ptr(<tng_dynamic_len_stack)->stack[__dynamic_len_idx] * sizeof(_type) * CHAR_BIT; \ lib_ring_buffer_align_ctx(&__ctx, lttng_alignof(_length_type));\ __chan->ops->event_write(&__ctx, &__tmpl, sizeof(_length_type));\ } \ @@ -882,7 +898,7 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar) \ /* Beware: this get len actually consumes the len value */ #undef __get_dynamic_len -#define __get_dynamic_len(field) __stackvar.__dynamic_len[__dynamic_len_idx++] +#define __get_dynamic_len(field) this_cpu_ptr(<tng_dynamic_len_stack)->stack[__dynamic_len_idx++] #undef TP_PROTO #define TP_PROTO(...) __VA_ARGS__ @@ -933,10 +949,11 @@ static void __event_probe__##_name(void *__data, _proto) \ struct lttng_channel *__chan = __event->chan; \ struct lttng_session *__session = __chan->session; \ struct lib_ring_buffer_ctx __ctx; \ - size_t __event_len, __event_align; \ - size_t __dynamic_len_idx __attribute__((unused)) = 0; \ + ssize_t __event_len; \ + size_t __event_align; \ + size_t __orig_dynamic_len_offset, __dynamic_len_idx; \ union { \ - size_t __dynamic_len[ARRAY_SIZE(__event_fields___##_name)]; \ + size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ } __stackvar; \ int __ret; \ @@ -956,6 +973,8 @@ static void __event_probe__##_name(void *__data, _proto) \ __lpf = lttng_rcu_dereference(__session->pid_tracker); \ if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid))) \ return; \ + __orig_dynamic_len_offset = this_cpu_ptr(<tng_dynamic_len_stack)->offset; \ + __dynamic_len_idx = __orig_dynamic_len_offset; \ _code_pre \ if (unlikely(!list_empty(&__event->bytecode_runtime_head))) { \ struct lttng_bytecode_runtime *bc_runtime; \ @@ -971,8 +990,11 @@ static void __event_probe__##_name(void *__data, _proto) \ if (likely(!__filter_record)) \ goto __post; \ } \ - __event_len = __event_get_size__##_name(__stackvar.__dynamic_len, \ - tp_locvar, _args); \ + __event_len = __event_get_size__##_name(tp_locvar, _args); \ + if (unlikely(__event_len < 0)) { \ + lib_ring_buffer_lost_event_too_big(__chan->chan); \ + goto __post; \ + } \ __event_align = __event_get_align__##_name(tp_locvar, _args); \ lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ __event_align, -1); \ @@ -983,6 +1005,8 @@ static void __event_probe__##_name(void *__data, _proto) \ __chan->ops->event_commit(&__ctx); \ __post: \ _code_post \ + barrier(); /* use before un-reserve. */ \ + this_cpu_ptr(<tng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \ return; \ } @@ -999,10 +1023,11 @@ static void __event_probe__##_name(void *__data) \ struct lttng_channel *__chan = __event->chan; \ struct lttng_session *__session = __chan->session; \ struct lib_ring_buffer_ctx __ctx; \ - size_t __event_len, __event_align; \ - size_t __dynamic_len_idx __attribute__((unused)) = 0; \ + ssize_t __event_len; \ + size_t __event_align; \ + size_t __orig_dynamic_len_offset, __dynamic_len_idx; \ union { \ - size_t __dynamic_len[ARRAY_SIZE(__event_fields___##_name)]; \ + size_t __dynamic_len_removed[ARRAY_SIZE(__event_fields___##_name)]; \ char __filter_stack_data[2 * sizeof(unsigned long) * ARRAY_SIZE(__event_fields___##_name)]; \ } __stackvar; \ int __ret; \ @@ -1022,6 +1047,8 @@ static void __event_probe__##_name(void *__data) \ __lpf = lttng_rcu_dereference(__session->pid_tracker); \ if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid))) \ return; \ + __orig_dynamic_len_offset = this_cpu_ptr(<tng_dynamic_len_stack)->offset; \ + __dynamic_len_idx = __orig_dynamic_len_offset; \ _code_pre \ if (unlikely(!list_empty(&__event->bytecode_runtime_head))) { \ struct lttng_bytecode_runtime *bc_runtime; \ @@ -1037,7 +1064,11 @@ static void __event_probe__##_name(void *__data) \ if (likely(!__filter_record)) \ goto __post; \ } \ - __event_len = __event_get_size__##_name(__stackvar.__dynamic_len, tp_locvar); \ + __event_len = __event_get_size__##_name(tp_locvar); \ + if (unlikely(__event_len < 0)) { \ + lib_ring_buffer_lost_event_too_big(__chan->chan); \ + goto __post; \ + } \ __event_align = __event_get_align__##_name(tp_locvar); \ lib_ring_buffer_ctx_init(&__ctx, __chan->chan, &__lttng_probe_ctx, __event_len, \ __event_align, -1); \ @@ -1048,6 +1079,8 @@ static void __event_probe__##_name(void *__data) \ __chan->ops->event_commit(&__ctx); \ __post: \ _code_post \ + barrier(); /* use before un-reserve. */ \ + this_cpu_ptr(<tng_dynamic_len_stack)->offset = __orig_dynamic_len_offset; \ return; \ } -- 2.34.1