[lttng-modules.git] / lttng-context-perf-counters.c

/*
 * (C) Copyright	2009-2011 -
 * 		Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * LTTng performance monitoring counters (perf-counters) integration module.
 *
 * Dual LGPL v2.1/GPL v2 license.
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/perf_event.h>
#include <linux/list.h>
#include <linux/string.h>
#include "ltt-events.h"
#include "wrapper/ringbuffer/frontend_types.h"
#include "wrapper/vmalloc.h"
#include "ltt-tracer.h"

static
size_t perf_counter_get_size(size_t offset)
{
	size_t size = 0;

	size += lib_ring_buffer_align(offset, ltt_alignof(uint64_t));
	size += sizeof(uint64_t);
	return size;
}

static
void perf_counter_record(struct lttng_ctx_field *field,
			 struct lib_ring_buffer_ctx *ctx,
			 struct ltt_channel *chan)
{
	struct perf_event *event;
	uint64_t value;

	event = field->u.perf_counter->e[ctx->cpu];
	if (likely(event)) {
		if (unlikely(event->state == PERF_EVENT_STATE_ERROR)) {
			value = 0;
		} else {
			event->pmu->read(event);
			value = local64_read(&event->count);
		}
	} else {
		/*
		 * Perf chooses not to be clever and not to support enabling a
		 * perf counter before the cpu is brought up. Therefore, we need
		 * to support having events coming (e.g. scheduler events)
		 * before the counter is setup. Write an arbitrary 0 in this
		 * case.
		 */
		value = 0;
	}
	lib_ring_buffer_align_ctx(ctx, ltt_alignof(value));
	chan->ops->event_write(ctx, &value, sizeof(value));
}

static
void overflow_callback(struct perf_event *event, int nmi,
		       struct perf_sample_data *data,
		       struct pt_regs *regs)
{
}

static
void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
{
	struct perf_event **events = field->u.perf_counter->e;
	int cpu;

	get_online_cpus();
	for_each_online_cpu(cpu)
		perf_event_release_kernel(events[cpu]);
	put_online_cpus();
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu_notifier(&field->u.perf_counter->nb);
#endif
	kfree(field->event_field.name);
	kfree(field->u.perf_counter->attr);
	kfree(events);
	kfree(field->u.perf_counter);
}

#ifdef CONFIG_HOTPLUG_CPU

/**
 *	lttng_perf_counter_hp_callback - CPU hotplug callback
 *	@nb: notifier block
 *	@action: hotplug action to take
 *	@hcpu: CPU number
 *
 *	Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
 *
 * We can setup perf counters when the cpu is online (up prepare seems to be too
 * soon).
 */
static
int __cpuinit lttng_perf_counter_cpu_hp_callback(struct notifier_block *nb,
						 unsigned long action,
						 void *hcpu)
{
	unsigned int cpu = (unsigned long) hcpu;
	struct lttng_perf_counter_field *perf_field =
		container_of(nb, struct lttng_perf_counter_field, nb);
	struct perf_event **events = perf_field->e;
	struct perf_event_attr *attr = perf_field->attr;
	struct perf_event *pevent;

	if (!perf_field->hp_enable)
		return NOTIFY_OK;

	switch (action) {
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		pevent = perf_event_create_kernel_counter(attr,
				cpu, NULL, overflow_callback);
		if (!pevent || IS_ERR(pevent))
			return NOTIFY_BAD;
		if (pevent->state == PERF_EVENT_STATE_ERROR) {
			perf_event_release_kernel(pevent);
			return NOTIFY_BAD;
		}
		barrier();	/* Create perf counter before setting event */
		events[cpu] = pevent;
		break;
	case CPU_UP_CANCELED:
	case CPU_UP_CANCELED_FROZEN:
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
		pevent = events[cpu];
		events[cpu] = NULL;
		barrier();	/* NULLify event before perf counter teardown */
		perf_event_release_kernel(pevent);
		break;
	}
	return NOTIFY_OK;
}

#endif

int lttng_add_perf_counter_to_ctx(uint32_t type,
				  uint64_t config,
				  const char *name,
				  struct lttng_ctx **ctx)
{
	struct lttng_ctx_field *field;
	struct lttng_perf_counter_field *perf_field;
	struct perf_event **events;
	struct perf_event_attr *attr;
	int ret;
	int cpu;
	char *name_alloc;

	events = kzalloc(num_possible_cpus() * sizeof(*events), GFP_KERNEL);
	if (!events)
		return -ENOMEM;

	attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL);
	if (!attr) {
		ret = -ENOMEM;
		goto error_attr;
	}

	attr->type = type;
	attr->config = config;
	attr->size = sizeof(struct perf_event_attr);
	attr->pinned = 1;
	attr->disabled = 0;

	perf_field = kzalloc(sizeof(struct lttng_perf_counter_field), GFP_KERNEL);
	if (!perf_field) {
		ret = -ENOMEM;
		goto error_alloc_perf_field;
	}
	perf_field->e = events;
	perf_field->attr = attr;

	name_alloc = kstrdup(name, GFP_KERNEL);
	if (!name_alloc) {
		ret = -ENOMEM;
		goto name_alloc_error;
	}

	field = lttng_append_context(ctx);
	if (!field) {
		ret = -ENOMEM;
		goto append_context_error;
	}
	if (lttng_find_context(*ctx, name_alloc)) {
		ret = -EEXIST;
		goto find_error;
	}

#ifdef CONFIG_HOTPLUG_CPU
	perf_field->nb.notifier_call =
		lttng_perf_counter_cpu_hp_callback;
	perf_field->nb.priority = 0;
	register_cpu_notifier(&perf_field->nb);
#endif

	get_online_cpus();
	for_each_online_cpu(cpu) {
		events[cpu] = perf_event_create_kernel_counter(attr,
					cpu, NULL, overflow_callback);
		if (!events[cpu] || IS_ERR(events[cpu])) {
			ret = -EINVAL;
			goto counter_error;
		}
		if (events[cpu]->state == PERF_EVENT_STATE_ERROR) {
			ret = -EBUSY;
			goto counter_busy;
		}
	}
	put_online_cpus();

	field->destroy = lttng_destroy_perf_counter_field;

	field->event_field.name = name_alloc;
	field->event_field.type.atype = atype_integer;
	field->event_field.type.u.basic.integer.size = sizeof(unsigned long) * CHAR_BIT;
	field->event_field.type.u.basic.integer.alignment = ltt_alignof(unsigned long) * CHAR_BIT;
	field->event_field.type.u.basic.integer.signedness = is_signed_type(unsigned long);
	field->event_field.type.u.basic.integer.reverse_byte_order = 0;
	field->event_field.type.u.basic.integer.base = 10;
	field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
	field->get_size = perf_counter_get_size;
	field->record = perf_counter_record;
	field->u.perf_counter = perf_field;
	perf_field->hp_enable = 1;

	wrapper_vmalloc_sync_all();
	return 0;

counter_busy:
counter_error:
	for_each_online_cpu(cpu) {
		if (events[cpu] && !IS_ERR(events[cpu]))
			perf_event_release_kernel(events[cpu]);
	}
	put_online_cpus();
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu_notifier(&perf_field->nb);
#endif
find_error:
	lttng_remove_context_field(ctx, field);
append_context_error:
	kfree(name_alloc);
name_alloc_error:
	kfree(perf_field);
error_alloc_perf_field:
	kfree(attr);
error_attr:
	kfree(events);
	return ret;
}

MODULE_LICENSE("GPL and additional rights");
MODULE_AUTHOR("Mathieu Desnoyers");
MODULE_DESCRIPTION("Linux Trace Toolkit Perf Support");
Commit	Line	Data
833ad6a0 MD	1	/*
	2	* (C) Copyright 2009-2011 -
	3	* Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	4	*
	5	* LTTng performance monitoring counters (perf-counters) integration module.
	6	*
	7	* Dual LGPL v2.1/GPL v2 license.
	8	*/
	9
	10	#include <linux/module.h>
	11	#include <linux/slab.h>
	12	#include <linux/perf_event.h>
	13	#include <linux/list.h>
c24a0d71 MD	14	#include <linux/string.h>
	15	#include "ltt-events.h"
	16	#include "wrapper/ringbuffer/frontend_types.h"
	17	#include "wrapper/vmalloc.h"
	18	#include "ltt-tracer.h"
833ad6a0	19
f1676205 MD	20	static
	21	size_t perf_counter_get_size(size_t offset)
	22	{
	23	size_t size = 0;
	24
	25	size += lib_ring_buffer_align(offset, ltt_alignof(uint64_t));
	26	size += sizeof(uint64_t);
	27	return size;
	28	}
	29
833ad6a0 MD	30	static
	31	void perf_counter_record(struct lttng_ctx_field *field,
	32	struct lib_ring_buffer_ctx *ctx,
	33	struct ltt_channel *chan)
	34	{
	35	struct perf_event *event;
	36	uint64_t value;
	37
2001023e	38	event = field->u.perf_counter->e[ctx->cpu];
0478c519	39	if (likely(event)) {
7b745a96 MD	40	if (unlikely(event->state == PERF_EVENT_STATE_ERROR)) {
	41	value = 0;
	42	} else {
	43	event->pmu->read(event);
	44	value = local64_read(&event->count);
	45	}
f91fd73b MD	46	} else {
	47	/*
	48	* Perf chooses not to be clever and not to support enabling a
	49	* perf counter before the cpu is brought up. Therefore, we need
	50	* to support having events coming (e.g. scheduler events)
	51	* before the counter is setup. Write an arbitrary 0 in this
	52	* case.
	53	*/
	54	value = 0;
	55	}
9e7e4892	56	lib_ring_buffer_align_ctx(ctx, ltt_alignof(value));
833ad6a0 MD	57	chan->ops->event_write(ctx, &value, sizeof(value));
	58	}
	59
	60	static
	61	void overflow_callback(struct perf_event *event, int nmi,
	62	struct perf_sample_data *data,
	63	struct pt_regs *regs)
	64	{
	65	}
	66
2dccf128 MD	67	static
	68	void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
	69	{
2001023e	70	struct perf_event **events = field->u.perf_counter->e;
2dccf128 MD	71	int cpu;
2dccf128 MD	72
8289661d	73	get_online_cpus();
2dccf128 MD	74	for_each_online_cpu(cpu)
2dccf128 MD	75	perf_event_release_kernel(events[cpu]);
8289661d MD	76	put_online_cpus();
8289661d MD	77	#ifdef CONFIG_HOTPLUG_CPU
2001023e	78	unregister_cpu_notifier(&field->u.perf_counter->nb);
8289661d	79	#endif
c24a0d71	80	kfree(field->event_field.name);
2001023e	81	kfree(field->u.perf_counter->attr);
2dccf128	82	kfree(events);
2001023e	83	kfree(field->u.perf_counter);
2dccf128 MD	84	}
2dccf128 MD	85
8289661d MD	86	#ifdef CONFIG_HOTPLUG_CPU
	87
	88	/**
	89	* lttng_perf_counter_hp_callback - CPU hotplug callback
	90	* @nb: notifier block
	91	* @action: hotplug action to take
	92	* @hcpu: CPU number
	93	*
	94	* Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
	95	*
	96	* We can setup perf counters when the cpu is online (up prepare seems to be too
	97	* soon).
	98	*/
	99	static
	100	int __cpuinit lttng_perf_counter_cpu_hp_callback(struct notifier_block *nb,
	101	unsigned long action,
	102	void *hcpu)
	103	{
	104	unsigned int cpu = (unsigned long) hcpu;
2001023e MD	105	struct lttng_perf_counter_field *perf_field =
	106	container_of(nb, struct lttng_perf_counter_field, nb);
	107	struct perf_event **events = perf_field->e;
	108	struct perf_event_attr *attr = perf_field->attr;
f91fd73b	109	struct perf_event *pevent;
8289661d	110
2001023e	111	if (!perf_field->hp_enable)
8289661d MD	112	return NOTIFY_OK;
	113
	114	switch (action) {
	115	case CPU_ONLINE:
	116	case CPU_ONLINE_FROZEN:
f91fd73b	117	pevent = perf_event_create_kernel_counter(attr,
8289661d	118	cpu, NULL, overflow_callback);
0478c519	119	if (!pevent \|\| IS_ERR(pevent))
8289661d	120	return NOTIFY_BAD;
7b745a96 MD	121	if (pevent->state == PERF_EVENT_STATE_ERROR) {
	122	perf_event_release_kernel(pevent);
	123	return NOTIFY_BAD;
	124	}
f91fd73b MD	125	barrier(); /* Create perf counter before setting event */
f91fd73b MD	126	events[cpu] = pevent;
8289661d MD	127	break;
	128	case CPU_UP_CANCELED:
	129	case CPU_UP_CANCELED_FROZEN:
	130	case CPU_DEAD:
	131	case CPU_DEAD_FROZEN:
f91fd73b MD	132	pevent = events[cpu];
	133	events[cpu] = NULL;
	134	barrier(); /* NULLify event before perf counter teardown */
	135	perf_event_release_kernel(pevent);
8289661d MD	136	break;
	137	}
	138	return NOTIFY_OK;
	139	}
	140
	141	#endif
	142
833ad6a0 MD	143	int lttng_add_perf_counter_to_ctx(uint32_t type,
833ad6a0 MD	144	uint64_t config,
c24a0d71	145	const char *name,
2dccf128	146	struct lttng_ctx **ctx)
833ad6a0 MD	147	{
833ad6a0 MD	148	struct lttng_ctx_field *field;
2001023e	149	struct lttng_perf_counter_field *perf_field;
833ad6a0 MD	150	struct perf_event **events;
	151	struct perf_event_attr *attr;
	152	int ret;
	153	int cpu;
c24a0d71	154	char *name_alloc;
833ad6a0 MD	155
	156	events = kzalloc(num_possible_cpus() * sizeof(*events), GFP_KERNEL);
	157	if (!events)
	158	return -ENOMEM;
	159
2001023e	160	attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL);
833ad6a0 MD	161	if (!attr) {
	162	ret = -ENOMEM;
	163	goto error_attr;
	164	}
	165
	166	attr->type = type;
	167	attr->config = config;
	168	attr->size = sizeof(struct perf_event_attr);
	169	attr->pinned = 1;
	170	attr->disabled = 0;
	171
2001023e MD	172	perf_field = kzalloc(sizeof(struct lttng_perf_counter_field), GFP_KERNEL);
	173	if (!perf_field) {
	174	ret = -ENOMEM;
	175	goto error_alloc_perf_field;
	176	}
	177	perf_field->e = events;
	178	perf_field->attr = attr;
	179
c24a0d71	180	name_alloc = kstrdup(name, GFP_KERNEL);
bef96e48 MD	181	if (!name_alloc) {
bef96e48 MD	182	ret = -ENOMEM;
c24a0d71	183	goto name_alloc_error;
bef96e48	184	}
c24a0d71	185
2dccf128 MD	186	field = lttng_append_context(ctx);
	187	if (!field) {
	188	ret = -ENOMEM;
8289661d	189	goto append_context_error;
833ad6a0	190	}
2001023e	191	if (lttng_find_context(*ctx, name_alloc)) {
44252f0f MD	192	ret = -EEXIST;
	193	goto find_error;
	194	}
8289661d MD	195
8289661d MD	196	#ifdef CONFIG_HOTPLUG_CPU
2001023e	197	perf_field->nb.notifier_call =
8289661d	198	lttng_perf_counter_cpu_hp_callback;
2001023e MD	199	perf_field->nb.priority = 0;
2001023e MD	200	register_cpu_notifier(&perf_field->nb);
8289661d MD	201	#endif
	202
	203	get_online_cpus();
	204	for_each_online_cpu(cpu) {
	205	events[cpu] = perf_event_create_kernel_counter(attr,
	206	cpu, NULL, overflow_callback);
0478c519	207	if (!events[cpu] \|\| IS_ERR(events[cpu])) {
8289661d MD	208	ret = -EINVAL;
	209	goto counter_error;
	210	}
7b745a96 MD	211	if (events[cpu]->state == PERF_EVENT_STATE_ERROR) {
	212	ret = -EBUSY;
	213	goto counter_busy;
	214	}
8289661d MD	215	}
	216	put_online_cpus();
	217
2dccf128	218	field->destroy = lttng_destroy_perf_counter_field;
833ad6a0	219
c24a0d71	220	field->event_field.name = name_alloc;
8070f5c0 MD	221	field->event_field.type.atype = atype_integer;
	222	field->event_field.type.u.basic.integer.size = sizeof(unsigned long) * CHAR_BIT;
	223	field->event_field.type.u.basic.integer.alignment = ltt_alignof(unsigned long) * CHAR_BIT;
	224	field->event_field.type.u.basic.integer.signedness = is_signed_type(unsigned long);
	225	field->event_field.type.u.basic.integer.reverse_byte_order = 0;
	226	field->event_field.type.u.basic.integer.base = 10;
	227	field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
f1676205 MD	228	field->get_size = perf_counter_get_size;
f1676205 MD	229	field->record = perf_counter_record;
2001023e MD	230	field->u.perf_counter = perf_field;
2001023e MD	231	perf_field->hp_enable = 1;
833ad6a0 MD	232
	233	wrapper_vmalloc_sync_all();
	234	return 0;
	235
7b745a96	236	counter_busy:
8289661d	237	counter_error:
833ad6a0	238	for_each_online_cpu(cpu) {
0478c519	239	if (events[cpu] && !IS_ERR(events[cpu]))
833ad6a0 MD	240	perf_event_release_kernel(events[cpu]);
833ad6a0 MD	241	}
8289661d MD	242	put_online_cpus();
8289661d MD	243	#ifdef CONFIG_HOTPLUG_CPU
2001023e	244	unregister_cpu_notifier(&perf_field->nb);
8289661d	245	#endif
44252f0f	246	find_error:
8289661d MD	247	lttng_remove_context_field(ctx, field);
	248	append_context_error:
	249	kfree(name_alloc);
	250	name_alloc_error:
2001023e MD	251	kfree(perf_field);
2001023e MD	252	error_alloc_perf_field:
833ad6a0 MD	253	kfree(attr);
	254	error_attr:
	255	kfree(events);
	256	return ret;
	257	}
	258
833ad6a0 MD	259	MODULE_LICENSE("GPL and additional rights");
	260	MODULE_AUTHOR("Mathieu Desnoyers");
	261	MODULE_DESCRIPTION("Linux Trace Toolkit Perf Support");