2 /* LTTng user-space "fast" tracing header
4 * Copyright 2006 Mathieu Desnoyers
8 #ifndef _LTT_USERTRACE_FAST_H
9 #define _LTT_USERTRACE_FAST_H
14 #include <asm/atomic.h>
18 #include <linux/futex.h>
20 #include <ltt/ltt-facility-id-user_generic.h>
21 #include <ltt/ltt-generic.h>
24 static inline __attribute__((no_instrument_function
))
25 _syscall6(long, futex
, unsigned long, uaddr
, int, op
, int, val
,
26 unsigned long, timeout
, unsigned long, uaddr2
, int, val2
)
32 #define LTT_N_SUBBUFS 2
33 #endif //LTT_N_SUBBUFS
35 #ifndef LTT_SUBBUF_SIZE_CPU
36 #define LTT_SUBBUF_SIZE_CPU 1048576
37 #endif //LTT_BUF_SIZE_CPU
39 #define LTT_BUF_SIZE_CPU (LTT_SUBBUF_SIZE_CPU * LTT_N_SUBBUFS)
41 #ifndef LTT_SUBBUF_SIZE_FACILITIES
42 #define LTT_SUBBUF_SIZE_FACILITIES 4096
43 #endif //LTT_BUF_SIZE_FACILITIES
45 #define LTT_BUF_SIZE_FACILITIES (LTT_SUBBUF_SIZE_FACILITIES * LTT_N_SUBBUFS)
47 #ifndef LTT_USERTRACE_ROOT
48 #define LTT_USERTRACE_ROOT "/tmp/ltt-usertrace"
49 #endif //LTT_USERTRACE_ROOT
52 /* Buffer offset macros */
54 #define BUFFER_OFFSET(offset, buf) (offset & (buf->alloc_size-1))
55 #define SUBBUF_OFFSET(offset, buf) (offset & (buf->subbuf_size-1))
56 #define SUBBUF_ALIGN(offset, buf) \
57 (((offset) + buf->subbuf_size) & (~(buf->subbuf_size-1)))
58 #define SUBBUF_TRUNC(offset, buf) \
59 ((offset) & (~(buf->subbuf_size-1)))
60 #define SUBBUF_INDEX(offset, buf) \
61 (BUFFER_OFFSET(offset,buf)/buf->subbuf_size)
64 #define LTT_TRACER_MAGIC_NUMBER 0x00D6B7ED
65 #define LTT_TRACER_VERSION_MAJOR 0
66 #define LTT_TRACER_VERSION_MINOR 7
68 #ifndef atomic_cmpxchg
69 #define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
70 #endif //atomic_cmpxchg
72 typedef unsigned int ltt_facility_t
;
74 struct ltt_trace_header
{
75 uint32_t magic_number
;
77 uint32_t arch_variant
;
78 uint32_t float_word_order
; /* Only useful for user space traces */
80 //uint32_t system_type;
81 uint8_t major_version
;
82 uint8_t minor_version
;
83 uint8_t flight_recorder
;
84 uint8_t has_heartbeat
;
85 uint8_t has_alignment
; /* Event header alignment */
89 uint64_t start_monotonic
;
90 uint64_t start_time_sec
;
91 uint64_t start_time_usec
;
92 } __attribute((packed
));
95 struct ltt_block_start_header
{
98 uint64_t freq
; /* khz */
101 uint64_t cycle_count
;
102 uint64_t freq
; /* khz */
104 uint32_t lost_size
; /* Size unused at the end of the buffer */
105 uint32_t buf_size
; /* The size of this sub-buffer */
106 struct ltt_trace_header trace
;
107 } __attribute((packed
));
115 atomic_t reserve_count
[LTT_N_SUBBUFS
];
116 atomic_t commit_count
[LTT_N_SUBBUFS
];
118 atomic_t events_lost
;
119 atomic_t corrupted_subbuffers
;
120 atomic_t full
; /* futex on which the writer waits : 1 : full */
121 unsigned int alloc_size
;
122 unsigned int subbuf_size
;
125 struct ltt_trace_info
{
131 struct ltt_buf facilities
;
133 char facilities_buf
[LTT_BUF_SIZE_FACILITIES
] __attribute__ ((aligned (8)));
134 char cpu_buf
[LTT_BUF_SIZE_CPU
] __attribute__ ((aligned (8)));
139 struct ltt_event_header_nohb
{
141 unsigned char facility_id
;
142 unsigned char event_id
;
144 } __attribute((packed
));
146 extern __thread
struct ltt_trace_info
*thread_trace_info
;
148 void ltt_thread_init(void);
150 void __attribute__((no_instrument_function
))
151 ltt_usertrace_fast_buffer_switch(void);
153 /* Get the offset of the channel in the ltt_trace_struct */
154 #define GET_CHANNEL_INDEX(chan) \
155 (unsigned int)&((struct ltt_trace_info*)NULL)->channel.chan
157 /* ltt_get_index_from_facility
159 * Get channel index from facility and event id.
162 * @eID : event number
164 * Get the channel index into which events must be written for the given
165 * facility and event number. We get this structure offset as soon as possible
166 * and remember it so we pass through this logic only once per trace call (not
169 static inline unsigned int __attribute__((no_instrument_function
))
170 ltt_get_index_from_facility(ltt_facility_t fID
,
173 return GET_CHANNEL_INDEX(cpu
);
177 static inline struct ltt_buf
* __attribute__((no_instrument_function
))
178 ltt_get_channel_from_index(
179 struct ltt_trace_info
*trace
, unsigned int index
)
181 return *(struct ltt_buf
**)((void*)trace
+index
);
186 * ltt_get_header_size
188 * Calculate alignment offset for arch size void*. This is the
189 * alignment offset of the event header.
192 * The event header must be a size multiple of the void* size. This is necessary
193 * to be able to calculate statically the alignment offset of the variable
194 * length data fields that follows. The total offset calculated here :
196 * Alignment of header struct on arch size
197 * + sizeof(header struct)
198 * + padding added to end of struct to align on arch size.
200 static inline unsigned char __attribute__((no_instrument_function
))
201 ltt_get_header_size(struct ltt_trace_info
*trace
,
203 size_t *before_hdr_pad
,
204 size_t *after_hdr_pad
,
207 unsigned int padding
;
210 header
= sizeof(struct ltt_event_header_nohb
);
212 /* Padding before the header. Calculated dynamically */
213 *before_hdr_pad
= ltt_align((unsigned long)address
, header
);
214 padding
= *before_hdr_pad
;
216 /* Padding after header, considering header aligned on ltt_align.
217 * Calculated statically if header size if known. */
218 *after_hdr_pad
= ltt_align(header
, sizeof(void*));
219 padding
+= *after_hdr_pad
;
221 *header_size
= header
;
223 return header
+padding
;
227 /* ltt_write_event_header
229 * Writes the event header to the pointer.
231 * @channel : pointer to the channel structure
232 * @ptr : buffer pointer
235 * @event_size : size of the event, excluding the event header.
236 * @offset : offset of the beginning of the header, for alignment.
237 * Calculated by ltt_get_event_header_size.
238 * @tsc : time stamp counter.
240 static inline void __attribute__((no_instrument_function
))
241 ltt_write_event_header(
242 struct ltt_trace_info
*trace
, struct ltt_buf
*buf
,
243 void *ptr
, ltt_facility_t fID
, uint32_t eID
, size_t event_size
,
244 size_t offset
, uint64_t tsc
)
246 struct ltt_event_header_nohb
*nohb
;
248 event_size
= min(event_size
, 0xFFFFU
);
249 nohb
= (struct ltt_event_header_nohb
*)(ptr
+offset
);
250 nohb
->timestamp
= (uint64_t)tsc
;
251 nohb
->facility_id
= fID
;
252 nohb
->event_id
= eID
;
253 nohb
->event_size
= (uint16_t)event_size
;
258 static inline uint64_t __attribute__((no_instrument_function
))
264 static inline unsigned int __attribute__((no_instrument_function
))
265 ltt_subbuf_header_len(struct ltt_buf
*buf
)
267 return sizeof(struct ltt_block_start_header
);
272 static inline void __attribute__((no_instrument_function
))
273 ltt_write_trace_header(struct ltt_trace_header
*header
)
275 header
->magic_number
= LTT_TRACER_MAGIC_NUMBER
;
276 header
->major_version
= LTT_TRACER_VERSION_MAJOR
;
277 header
->minor_version
= LTT_TRACER_VERSION_MINOR
;
278 header
->float_word_order
= 0; //FIXME
279 header
->arch_type
= 0; //FIXME LTT_ARCH_TYPE;
280 header
->arch_size
= sizeof(void*);
281 header
->arch_variant
= 0; //FIXME LTT_ARCH_VARIANT;
282 header
->flight_recorder
= 0;
283 header
->has_heartbeat
= 0;
286 header
->has_alignment
= sizeof(void*);
288 header
->has_alignment
= 0;
292 header
->freq_scale
= 0;
293 header
->start_freq
= 0;
294 header
->start_tsc
= 0;
295 header
->start_monotonic
= 0;
296 header
->start_time_sec
= 0;
297 header
->start_time_usec
= 0;
301 static inline void __attribute__((no_instrument_function
))
302 ltt_buffer_begin_callback(struct ltt_buf
*buf
,
303 uint64_t tsc
, unsigned int subbuf_idx
)
305 struct ltt_block_start_header
*header
=
306 (struct ltt_block_start_header
*)
307 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
309 header
->begin
.cycle_count
= tsc
;
310 header
->begin
.freq
= 0; //ltt_frequency();
312 header
->lost_size
= 0xFFFFFFFF; // for debugging...
314 header
->buf_size
= buf
->subbuf_size
;
316 ltt_write_trace_header(&header
->trace
);
322 static inline void __attribute__((no_instrument_function
))
323 ltt_buffer_end_callback(struct ltt_buf
*buf
,
324 uint64_t tsc
, unsigned int offset
, unsigned int subbuf_idx
)
326 struct ltt_block_start_header
*header
=
327 (struct ltt_block_start_header
*)
328 (buf
->start
+ (subbuf_idx
*buf
->subbuf_size
));
329 /* offset is assumed to never be 0 here : never deliver a completely
330 * empty subbuffer. */
331 /* The lost size is between 0 and subbuf_size-1 */
332 header
->lost_size
= SUBBUF_OFFSET((buf
->subbuf_size
- offset
),
334 header
->end
.cycle_count
= tsc
;
335 header
->end
.freq
= 0; //ltt_frequency();
339 static inline void __attribute__((no_instrument_function
))
340 ltt_deliver_callback(struct ltt_buf
*buf
,
344 ltt_usertrace_fast_buffer_switch();
350 * Atomic slot reservation in a LTTng buffer. It will take care of
351 * sub-buffer switching.
355 * @trace : the trace structure to log to.
356 * @buf : the buffer to reserve space into.
357 * @data_size : size of the variable length data to log.
358 * @slot_size : pointer to total size of the slot (out)
359 * @tsc : pointer to the tsc at the slot reservation (out)
360 * @before_hdr_pad : dynamic padding before the event header.
361 * @after_hdr_pad : dynamic padding after the event header.
363 * Return : NULL if not enough space, else returns the pointer
364 * to the beginning of the reserved slot. */
365 static inline void * __attribute__((no_instrument_function
)) ltt_reserve_slot(
366 struct ltt_trace_info
*trace
,
367 struct ltt_buf
*ltt_buf
,
368 unsigned int data_size
,
369 unsigned int *slot_size
,
371 size_t *before_hdr_pad
,
372 size_t *after_hdr_pad
,
375 int offset_begin
, offset_end
, offset_old
;
377 int begin_switch
, end_switch_current
, end_switch_old
;
378 int reserve_commit_diff
= 0;
380 int consumed_old
, consumed_new
;
381 int commit_count
, reserve_count
;
385 offset_old
= atomic_read(<t_buf
->offset
);
386 offset_begin
= offset_old
;
389 end_switch_current
= 0;
391 *tsc
= ltt_get_timestamp();
393 /* Error in getting the timestamp, event lost */
394 atomic_inc(<t_buf
->events_lost
);
398 if(SUBBUF_OFFSET(offset_begin
, ltt_buf
) == 0) {
399 begin_switch
= 1; /* For offset_begin */
401 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
402 before_hdr_pad
, after_hdr_pad
, header_size
)
405 if((SUBBUF_OFFSET(offset_begin
, ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
407 end_switch_old
= 1; /* For offset_old */
408 begin_switch
= 1; /* For offset_begin */
414 offset_begin
= SUBBUF_ALIGN(offset_begin
, ltt_buf
);
416 offset_begin
= offset_begin
+ ltt_subbuf_header_len(ltt_buf
);
417 /* Test new buffer integrity */
418 reserve_commit_diff
=
419 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
421 - atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
423 if(reserve_commit_diff
== 0) {
424 /* Next buffer not corrupted. */
425 if((SUBBUF_TRUNC(offset_begin
, ltt_buf
)
426 - SUBBUF_TRUNC(atomic_read(<t_buf
->consumed
), ltt_buf
))
427 >= ltt_buf
->alloc_size
) {
428 /* We block until the reader unblocks us */
429 atomic_set(<t_buf
->full
, 1);
430 /* We block until the reader tells us to wake up.
431 Signals will simply cause this loop to restart.
434 ret
= futex((unsigned long)<t_buf
->full
, FUTEX_WAIT
, 1, 0, 0, 0);
435 } while(ret
!= 0 && ret
!= EWOULDBLOCK
);
436 /* go on with the write */
439 /* next buffer not corrupted, we are either in overwrite mode or
440 * the buffer is not full. It's safe to write in this new subbuffer.*/
443 /* Next subbuffer corrupted. Force pushing reader even in normal
444 * mode. It's safe to write in this new subbuffer. */
446 size
= ltt_get_header_size(trace
, ltt_buf
->start
+ offset_begin
,
447 before_hdr_pad
, after_hdr_pad
, header_size
) + data_size
;
448 if((SUBBUF_OFFSET(offset_begin
,ltt_buf
)+size
)>ltt_buf
->subbuf_size
) {
449 /* Event too big for subbuffers, report error, don't complete
450 * the sub-buffer switch. */
451 atomic_inc(<t_buf
->events_lost
);
454 /* We just made a successful buffer switch and the event fits in the
455 * new subbuffer. Let's write. */
458 /* Event fits in the current buffer and we are not on a switch boundary.
459 * It's safe to write */
461 offset_end
= offset_begin
+ size
;
463 if((SUBBUF_OFFSET(offset_end
, ltt_buf
)) == 0) {
464 /* The offset_end will fall at the very beginning of the next subbuffer.
466 end_switch_current
= 1; /* For offset_begin */
469 } while(atomic_cmpxchg(<t_buf
->offset
, offset_old
, offset_end
)
473 /* Push the reader if necessary */
475 consumed_old
= atomic_read(<t_buf
->consumed
);
476 /* If buffer is in overwrite mode, push the reader consumed count if
477 the write position has reached it and we are not at the first
478 iteration (don't push the reader farther than the writer).
479 This operation can be done concurrently by many writers in the
480 same buffer, the writer being at the fartest write position sub-buffer
481 index in the buffer being the one which will win this loop. */
482 /* If the buffer is not in overwrite mode, pushing the reader only
483 happen if a sub-buffer is corrupted */
484 if((SUBBUF_TRUNC(offset_end
, ltt_buf
)
485 - SUBBUF_TRUNC(consumed_old
, ltt_buf
))
486 >= ltt_buf
->alloc_size
)
487 consumed_new
= SUBBUF_ALIGN(consumed_old
, ltt_buf
);
489 consumed_new
= consumed_old
;
492 } while(atomic_cmpxchg(<t_buf
->consumed
, consumed_old
, consumed_new
)
495 if(consumed_old
!= consumed_new
) {
496 /* Reader pushed : we are the winner of the push, we can therefore
497 reequilibrate reserve and commit. Atomic increment of the commit
498 count permits other writers to play around with this variable
499 before us. We keep track of corrupted_subbuffers even in overwrite mode :
500 we never want to write over a non completely committed sub-buffer :
501 possible causes : the buffer size is too low compared to the unordered
502 data input, or there is a writer who died between the reserve and the
504 if(reserve_commit_diff
) {
505 /* We have to alter the sub-buffer commit count : a sub-buffer is
506 corrupted. We do not deliver it. */
507 atomic_add(reserve_commit_diff
,
508 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
509 atomic_inc(<t_buf
->corrupted_subbuffers
);
516 /* Concurrency safe because we are the last and only thread to alter this
517 sub-buffer. As long as it is not delivered and read, no other thread can
518 alter the offset, alter the reserve_count or call the
519 client_buffer_end_callback on this sub-buffer.
520 The only remaining threads could be the ones with pending commits. They
521 will have to do the deliver themself.
522 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
523 with commit and reserve counts. We keep a corrupted sub-buffers count
524 and push the readers across these sub-buffers.
525 Not concurrency safe if a writer is stalled in a subbuffer and
526 another writer switches in, finding out it's corrupted. The result will
527 be than the old (uncommited) subbuffer will be declared corrupted, and
528 that the new subbuffer will be declared corrupted too because of the
529 commit count adjustment.
530 Note : offset_old should never be 0 here.*/
531 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_old
,
532 SUBBUF_INDEX((offset_old
-1), ltt_buf
));
533 /* Setting this reserve_count will allow the sub-buffer to be delivered by
534 the last committer. */
536 atomic_add_return((SUBBUF_OFFSET((offset_old
-1), ltt_buf
)+1),
537 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_old
-1), ltt_buf
)]);
539 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_old
-1),
541 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_old
-1), ltt_buf
),
548 /* This code can be executed unordered : writers may already have written
549 to the sub-buffer before this code gets executed, caution. */
550 /* The commit makes sure that this code is executed before the deliver
551 of this sub-buffer */
552 ltt_buffer_begin_callback(ltt_buf
, *tsc
, SUBBUF_INDEX(offset_begin
, ltt_buf
));
553 commit_count
= atomic_add_return(ltt_subbuf_header_len(ltt_buf
),
554 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)]);
555 /* Check if the written buffer has to be delivered */
557 == atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
,
559 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
563 if(end_switch_current
) {
564 /* current subbuffer */
565 /* Concurrency safe because we are the last and only thread to alter this
566 sub-buffer. As long as it is not delivered and read, no other thread can
567 alter the offset, alter the reserve_count or call the
568 client_buffer_end_callback on this sub-buffer.
569 The only remaining threads could be the ones with pending commits. They
570 will have to do the deliver themself.
571 Not concurrency safe in overwrite mode. We detect corrupted subbuffers
572 with commit and reserve counts. We keep a corrupted sub-buffers count
573 and push the readers across these sub-buffers.
574 Not concurrency safe if a writer is stalled in a subbuffer and
575 another writer switches in, finding out it's corrupted. The result will
576 be than the old (uncommited) subbuffer will be declared corrupted, and
577 that the new subbuffer will be declared corrupted too because of the
578 commit count adjustment. */
579 ltt_buffer_end_callback(ltt_buf
, *tsc
, offset_end
,
580 SUBBUF_INDEX((offset_end
-1), ltt_buf
));
581 /* Setting this reserve_count will allow the sub-buffer to be delivered by
582 the last committer. */
584 atomic_add_return((SUBBUF_OFFSET((offset_end
-1), ltt_buf
)+1),
585 <t_buf
->reserve_count
[SUBBUF_INDEX((offset_end
-1), ltt_buf
)]);
587 == atomic_read(<t_buf
->commit_count
[SUBBUF_INDEX((offset_end
-1),
589 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX((offset_end
-1), ltt_buf
), NULL
);
595 //BUG_ON(*slot_size != (data_size + *before_hdr_pad + *after_hdr_pad + *header_size));
596 //BUG_ON(*slot_size != (offset_end - offset_begin));
598 return ltt_buf
->start
+ BUFFER_OFFSET(offset_begin
, ltt_buf
);
604 * Atomic unordered slot commit. Increments the commit count in the
605 * specified sub-buffer, and delivers it if necessary.
609 * @buf : the buffer to commit to.
610 * @reserved : address of the beginnig of the reserved slot.
611 * @slot_size : size of the reserved slot.
614 static inline void __attribute__((no_instrument_function
)) ltt_commit_slot(
615 struct ltt_buf
*ltt_buf
,
617 unsigned int slot_size
)
619 unsigned int offset_begin
= reserved
- ltt_buf
->start
;
622 commit_count
= atomic_add_return(slot_size
,
623 <t_buf
->commit_count
[SUBBUF_INDEX(offset_begin
,
626 /* Check if all commits have been done */
628 atomic_read(<t_buf
->reserve_count
[SUBBUF_INDEX(offset_begin
, ltt_buf
)])) {
629 ltt_deliver_callback(ltt_buf
, SUBBUF_INDEX(offset_begin
, ltt_buf
), NULL
);
637 #endif //_LTT_USERTRACE_FAST_H