2 * lttng-context-perf-counters.c
4 * LTTng UST performance monitoring counters (perf-counters) integration.
6 * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
8 * This library is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; only
11 * version 2.1 of the License.
13 * This library is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with this library; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 #include <sys/types.h>
30 #include <sys/syscall.h>
31 #include <linux/perf_event.h>
32 #include <lttng/ust-events.h>
33 #include <lttng/ust-tracer.h>
34 #include <lttng/ringbuffer-config.h>
35 #include <urcu/system.h>
36 #include <urcu/arch.h>
37 #include <urcu/rculist.h>
40 #include <usterr-signal-safe.h>
42 #include "lttng-tracer-core.h"
45 * We use a global perf counter key and iterate on per-thread RCU lists
46 * of fields in the fast path, even though this is not strictly speaking
47 * what would provide the best fast-path complexity, to ensure teardown
48 * of sessions vs thread exit is handled racelessly.
50 * Updates and traversals of thread_list are protected by UST lock.
51 * Updates to rcu_field_list are protected by UST lock.
54 struct lttng_perf_counter_thread_field
{
55 struct lttng_perf_counter_field
*field
; /* Back reference */
56 struct perf_event_mmap_page
*pc
;
57 struct cds_list_head thread_field_node
; /* Per-field list of thread fields (node) */
58 struct cds_list_head rcu_field_node
; /* RCU per-thread list of fields (node) */
62 struct lttng_perf_counter_thread
{
63 struct cds_list_head rcu_field_list
; /* RCU per-thread list of fields */
66 struct lttng_perf_counter_field
{
67 struct perf_event_attr attr
;
68 struct cds_list_head thread_field_list
; /* Per-field list of thread fields */
71 static pthread_key_t perf_counter_key
;
74 size_t perf_counter_get_size(struct lttng_ctx_field
*field
, size_t offset
)
78 size
+= lib_ring_buffer_align(offset
, lttng_alignof(uint64_t));
79 size
+= sizeof(uint64_t);
83 #if defined(__x86_64__) || defined(__i386__)
86 uint64_t rdpmc(unsigned int counter
)
88 unsigned int low
, high
;
90 asm volatile("rdpmc" : "=a" (low
), "=d" (high
) : "c" (counter
));
92 return low
| ((uint64_t) high
) << 32;
95 static bool arch_perf_use_read(void)
101 uint64_t read_perf_counter(
102 struct lttng_perf_counter_thread_field
*thread_field
)
106 struct perf_event_mmap_page
*pc
= thread_field
->pc
;
108 if (caa_unlikely(!pc
))
112 seq
= CMM_LOAD_SHARED(pc
->lock
);
119 pmcval
= rdpmc(idx
- 1);
120 /* Sign-extend the pmc register result. */
121 pmcval
<<= 64 - pc
->pmc_width
;
122 pmcval
>>= 64 - pc
->pmc_width
;
123 count
= pc
->offset
+ pmcval
;
128 } while (CMM_LOAD_SHARED(pc
->lock
) != seq
);
133 #elif defined (__ARM_ARCH_7A__)
135 static bool arch_perf_use_read(void)
141 uint64_t read_perf_counter(
142 struct lttng_perf_counter_thread_field
*thread_field
)
146 if (caa_unlikely(thread_field
->fd
< 0))
149 if (caa_unlikely(read(thread_field
->fd
, &count
, sizeof(count
))
156 #else /* defined(__x86_64__) || defined(__i386__) || defined(__ARM_ARCH_7A__) */
158 #error "Perf event counters are only supported on x86 and ARMv7 so far."
160 #endif /* #else defined(__x86_64__) || defined(__i386__) || defined(__ARM_ARCH_7A__) */
163 int sys_perf_event_open(struct perf_event_attr
*attr
,
164 pid_t pid
, int cpu
, int group_fd
,
167 return syscall(SYS_perf_event_open
, attr
, pid
, cpu
,
172 int open_perf_fd(struct perf_event_attr
*attr
)
176 fd
= sys_perf_event_open(attr
, 0, -1, -1, 0);
184 void close_perf_fd(int fd
)
193 perror("Error closing LTTng-UST perf memory mapping FD");
198 struct perf_event_mmap_page
*setup_perf(
199 struct lttng_perf_counter_thread_field
*thread_field
)
203 perf_addr
= mmap(NULL
, sizeof(struct perf_event_mmap_page
),
204 PROT_READ
, MAP_SHARED
, thread_field
->fd
, 0);
205 if (perf_addr
== MAP_FAILED
)
208 if (!arch_perf_use_read()) {
209 close_perf_fd(thread_field
->fd
);
210 thread_field
->fd
= -1;
217 void unmap_perf_page(struct perf_event_mmap_page
*pc
)
223 ret
= munmap(pc
, sizeof(struct perf_event_mmap_page
));
225 PERROR("Error in munmap");
231 struct lttng_perf_counter_thread
*alloc_perf_counter_thread(void)
233 struct lttng_perf_counter_thread
*perf_thread
;
234 sigset_t newmask
, oldmask
;
237 ret
= sigfillset(&newmask
);
240 ret
= pthread_sigmask(SIG_BLOCK
, &newmask
, &oldmask
);
243 /* Check again with signals disabled */
244 perf_thread
= pthread_getspecific(perf_counter_key
);
247 perf_thread
= zmalloc(sizeof(*perf_thread
));
250 CDS_INIT_LIST_HEAD(&perf_thread
->rcu_field_list
);
251 ret
= pthread_setspecific(perf_counter_key
, perf_thread
);
255 ret
= pthread_sigmask(SIG_SETMASK
, &oldmask
, NULL
);
262 struct lttng_perf_counter_thread_field
*
263 add_thread_field(struct lttng_perf_counter_field
*perf_field
,
264 struct lttng_perf_counter_thread
*perf_thread
)
266 struct lttng_perf_counter_thread_field
*thread_field
;
267 sigset_t newmask
, oldmask
;
270 ret
= sigfillset(&newmask
);
273 ret
= pthread_sigmask(SIG_BLOCK
, &newmask
, &oldmask
);
276 /* Check again with signals disabled */
277 cds_list_for_each_entry_rcu(thread_field
, &perf_thread
->rcu_field_list
,
279 if (thread_field
->field
== perf_field
)
282 thread_field
= zmalloc(sizeof(*thread_field
));
285 thread_field
->field
= perf_field
;
286 thread_field
->fd
= open_perf_fd(&perf_field
->attr
);
287 if (thread_field
->fd
>= 0)
288 thread_field
->pc
= setup_perf(thread_field
);
290 * Note: thread_field->pc can be NULL if setup_perf() fails.
291 * Also, thread_field->fd can be -1 if open_perf_fd() fails.
294 cds_list_add_rcu(&thread_field
->rcu_field_node
,
295 &perf_thread
->rcu_field_list
);
296 cds_list_add(&thread_field
->thread_field_node
,
297 &perf_field
->thread_field_list
);
300 ret
= pthread_sigmask(SIG_SETMASK
, &oldmask
, NULL
);
307 struct lttng_perf_counter_thread_field
*
308 get_thread_field(struct lttng_perf_counter_field
*field
)
310 struct lttng_perf_counter_thread
*perf_thread
;
311 struct lttng_perf_counter_thread_field
*thread_field
;
313 perf_thread
= pthread_getspecific(perf_counter_key
);
315 perf_thread
= alloc_perf_counter_thread();
316 cds_list_for_each_entry_rcu(thread_field
, &perf_thread
->rcu_field_list
,
318 if (thread_field
->field
== field
)
321 /* perf_counter_thread_field not found, need to add one */
322 return add_thread_field(field
, perf_thread
);
326 uint64_t wrapper_perf_counter_read(struct lttng_ctx_field
*field
)
328 struct lttng_perf_counter_field
*perf_field
;
329 struct lttng_perf_counter_thread_field
*perf_thread_field
;
331 perf_field
= field
->u
.perf_counter
;
332 perf_thread_field
= get_thread_field(perf_field
);
333 return read_perf_counter(perf_thread_field
);
337 void perf_counter_record(struct lttng_ctx_field
*field
,
338 struct lttng_ust_lib_ring_buffer_ctx
*ctx
,
339 struct lttng_channel
*chan
)
343 value
= wrapper_perf_counter_read(field
);
344 lib_ring_buffer_align_ctx(ctx
, lttng_alignof(value
));
345 chan
->ops
->event_write(ctx
, &value
, sizeof(value
));
349 void perf_counter_get_value(struct lttng_ctx_field
*field
,
350 struct lttng_ctx_value
*value
)
354 v
= wrapper_perf_counter_read(field
);
358 /* Called with UST lock held */
360 void lttng_destroy_perf_thread_field(
361 struct lttng_perf_counter_thread_field
*thread_field
)
363 close_perf_fd(thread_field
->fd
);
364 unmap_perf_page(thread_field
->pc
);
365 cds_list_del_rcu(&thread_field
->rcu_field_node
);
366 cds_list_del(&thread_field
->thread_field_node
);
371 void lttng_destroy_perf_thread_key(void *_key
)
373 struct lttng_perf_counter_thread
*perf_thread
= _key
;
374 struct lttng_perf_counter_thread_field
*pos
, *p
;
377 cds_list_for_each_entry_safe(pos
, p
, &perf_thread
->rcu_field_list
,
379 lttng_destroy_perf_thread_field(pos
);
384 /* Called with UST lock held */
386 void lttng_destroy_perf_counter_field(struct lttng_ctx_field
*field
)
388 struct lttng_perf_counter_field
*perf_field
;
389 struct lttng_perf_counter_thread_field
*pos
, *p
;
391 free((char *) field
->event_field
.name
);
392 perf_field
= field
->u
.perf_counter
;
394 * This put is performed when no threads can concurrently
395 * perform a "get" concurrently, thanks to urcu-bp grace
398 cds_list_for_each_entry_safe(pos
, p
, &perf_field
->thread_field_list
,
400 lttng_destroy_perf_thread_field(pos
);
404 #ifdef __ARM_ARCH_7A__
407 int perf_get_exclude_kernel(void)
412 #else /* __ARM_ARCH_7A__ */
415 int perf_get_exclude_kernel(void)
420 #endif /* __ARM_ARCH_7A__ */
422 /* Called with UST lock held */
423 int lttng_add_perf_counter_to_ctx(uint32_t type
,
426 struct lttng_ctx
**ctx
)
428 struct lttng_ctx_field
*field
;
429 struct lttng_perf_counter_field
*perf_field
;
433 name_alloc
= strdup(name
);
436 goto name_alloc_error
;
438 perf_field
= zmalloc(sizeof(*perf_field
));
441 goto perf_field_alloc_error
;
443 field
= lttng_append_context(ctx
);
446 goto append_context_error
;
448 if (lttng_find_context(*ctx
, name_alloc
)) {
453 field
->destroy
= lttng_destroy_perf_counter_field
;
455 field
->event_field
.name
= name_alloc
;
456 field
->event_field
.type
.atype
= atype_integer
;
457 field
->event_field
.type
.u
.basic
.integer
.size
=
458 sizeof(uint64_t) * CHAR_BIT
;
459 field
->event_field
.type
.u
.basic
.integer
.alignment
=
460 lttng_alignof(uint64_t) * CHAR_BIT
;
461 field
->event_field
.type
.u
.basic
.integer
.signedness
=
462 lttng_is_signed_type(uint64_t);
463 field
->event_field
.type
.u
.basic
.integer
.reverse_byte_order
= 0;
464 field
->event_field
.type
.u
.basic
.integer
.base
= 10;
465 field
->event_field
.type
.u
.basic
.integer
.encoding
= lttng_encode_none
;
466 field
->get_size
= perf_counter_get_size
;
467 field
->record
= perf_counter_record
;
468 field
->get_value
= perf_counter_get_value
;
470 perf_field
->attr
.type
= type
;
471 perf_field
->attr
.config
= config
;
472 perf_field
->attr
.exclude_kernel
= perf_get_exclude_kernel();
473 CDS_INIT_LIST_HEAD(&perf_field
->thread_field_list
);
474 field
->u
.perf_counter
= perf_field
;
476 /* Ensure that this perf counter can be used in this process. */
477 ret
= open_perf_fd(&perf_field
->attr
);
485 * Contexts can only be added before tracing is started, so we
486 * don't have to synchronize against concurrent threads using
490 lttng_context_update(*ctx
);
495 lttng_remove_context_field(ctx
, field
);
496 append_context_error
:
498 perf_field_alloc_error
:
504 int lttng_perf_counter_init(void)
508 ret
= pthread_key_create(&perf_counter_key
,
509 lttng_destroy_perf_thread_key
);
515 void lttng_perf_counter_exit(void)
519 ret
= pthread_key_delete(perf_counter_key
);
522 PERROR("Error in pthread_key_delete");