Fix shadow channel struct callback override in consumer
[lttng-ust.git] / liblttng-ust / ltt-ring-buffer-client.h
1 /*
2 * ltt-ring-buffer-client.h
3 *
4 * Copyright (C) 2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
5 *
6 * LTTng lib ring buffer client template.
7 *
8 * Dual LGPL v2.1/GPL v2 license.
9 */
10
11 #include <stdint.h>
12 #include <lttng/ust-events.h>
13 #include "lttng/bitfield.h"
14 #include "lttng/clock.h"
15 #include "ltt-tracer.h"
16 #include "../libringbuffer/frontend_types.h"
17
18 /*
19 * Keep the natural field alignment for _each field_ within this structure if
20 * you ever add/remove a field from this header. Packed attribute is not used
21 * because gcc generates poor code on at least powerpc and mips. Don't ever
22 * let gcc add padding between the structure elements.
23 */
24
25 struct packet_header {
26 /* Trace packet header */
27 uint32_t magic; /*
28 * Trace magic number.
29 * contains endianness information.
30 */
31 uint8_t uuid[16];
32 uint32_t stream_id;
33
34 struct {
35 /* Stream packet context */
36 uint64_t timestamp_begin; /* Cycle count at subbuffer start */
37 uint64_t timestamp_end; /* Cycle count at subbuffer end */
38 uint32_t events_discarded; /*
39 * Events lost in this subbuffer since
40 * the beginning of the trace.
41 * (may overflow)
42 */
43 uint32_t content_size; /* Size of data in subbuffer */
44 uint32_t packet_size; /* Subbuffer size (include padding) */
45 uint32_t cpu_id; /* CPU id associated with stream */
46 uint8_t header_end; /* End of header */
47 } ctx;
48 };
49
50
51 static inline notrace u64 lib_ring_buffer_clock_read(struct channel *chan)
52 {
53 return trace_clock_read64();
54 }
55
56 static inline
57 size_t ctx_get_size(size_t offset, struct lttng_ctx *ctx)
58 {
59 int i;
60 size_t orig_offset = offset;
61
62 if (caa_likely(!ctx))
63 return 0;
64 for (i = 0; i < ctx->nr_fields; i++)
65 offset += ctx->fields[i].get_size(offset);
66 return offset - orig_offset;
67 }
68
69 static inline
70 void ctx_record(struct lttng_ust_lib_ring_buffer_ctx *bufctx,
71 struct ltt_channel *chan,
72 struct lttng_ctx *ctx)
73 {
74 int i;
75
76 if (caa_likely(!ctx))
77 return;
78 for (i = 0; i < ctx->nr_fields; i++)
79 ctx->fields[i].record(&ctx->fields[i], bufctx, chan);
80 }
81
82 /*
83 * record_header_size - Calculate the header size and padding necessary.
84 * @config: ring buffer instance configuration
85 * @chan: channel
86 * @offset: offset in the write buffer
87 * @pre_header_padding: padding to add before the header (output)
88 * @ctx: reservation context
89 *
90 * Returns the event header size (including padding).
91 *
92 * The payload must itself determine its own alignment from the biggest type it
93 * contains.
94 */
95 static __inline__
96 unsigned char record_header_size(const struct lttng_ust_lib_ring_buffer_config *config,
97 struct channel *chan, size_t offset,
98 size_t *pre_header_padding,
99 struct lttng_ust_lib_ring_buffer_ctx *ctx)
100 {
101 struct ltt_channel *ltt_chan = channel_get_private(chan);
102 struct ltt_event *event = ctx->priv;
103 size_t orig_offset = offset;
104 size_t padding;
105
106 switch (ltt_chan->header_type) {
107 case 1: /* compact */
108 padding = lib_ring_buffer_align(offset, lttng_alignof(uint32_t));
109 offset += padding;
110 if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTT_RFLAG_EXTENDED))) {
111 offset += sizeof(uint32_t); /* id and timestamp */
112 } else {
113 /* Minimum space taken by 5-bit id */
114 offset += sizeof(uint8_t);
115 /* Align extended struct on largest member */
116 offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
117 offset += sizeof(uint32_t); /* id */
118 offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
119 offset += sizeof(uint64_t); /* timestamp */
120 }
121 break;
122 case 2: /* large */
123 padding = lib_ring_buffer_align(offset, lttng_alignof(uint16_t));
124 offset += padding;
125 offset += sizeof(uint16_t);
126 if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTT_RFLAG_EXTENDED))) {
127 offset += lib_ring_buffer_align(offset, lttng_alignof(uint32_t));
128 offset += sizeof(uint32_t); /* timestamp */
129 } else {
130 /* Align extended struct on largest member */
131 offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
132 offset += sizeof(uint32_t); /* id */
133 offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
134 offset += sizeof(uint64_t); /* timestamp */
135 }
136 break;
137 default:
138 padding = 0;
139 WARN_ON_ONCE(1);
140 }
141 offset += ctx_get_size(offset, event->ctx);
142 offset += ctx_get_size(offset, ltt_chan->ctx);
143
144 *pre_header_padding = padding;
145 return offset - orig_offset;
146 }
147
148 #include "../libringbuffer/api.h"
149
150 static
151 void ltt_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config,
152 struct lttng_ust_lib_ring_buffer_ctx *ctx,
153 uint32_t event_id);
154
155 /*
156 * ltt_write_event_header
157 *
158 * Writes the event header to the offset (already aligned on 32-bits).
159 *
160 * @config: ring buffer instance configuration
161 * @ctx: reservation context
162 * @event_id: event ID
163 */
164 static __inline__
165 void ltt_write_event_header(const struct lttng_ust_lib_ring_buffer_config *config,
166 struct lttng_ust_lib_ring_buffer_ctx *ctx,
167 uint32_t event_id)
168 {
169 struct ltt_channel *ltt_chan = channel_get_private(ctx->chan);
170 struct ltt_event *event = ctx->priv;
171
172 if (caa_unlikely(ctx->rflags))
173 goto slow_path;
174
175 switch (ltt_chan->header_type) {
176 case 1: /* compact */
177 {
178 uint32_t id_time = 0;
179
180 bt_bitfield_write(&id_time, uint32_t, 0, 5, event_id);
181 bt_bitfield_write(&id_time, uint32_t, 5, 27, ctx->tsc);
182 lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time));
183 break;
184 }
185 case 2: /* large */
186 {
187 uint32_t timestamp = (uint32_t) ctx->tsc;
188 uint16_t id = event_id;
189
190 lib_ring_buffer_write(config, ctx, &id, sizeof(id));
191 lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint32_t));
192 lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
193 break;
194 }
195 default:
196 WARN_ON_ONCE(1);
197 }
198
199 ctx_record(ctx, ltt_chan, ltt_chan->ctx);
200 ctx_record(ctx, ltt_chan, event->ctx);
201 lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
202
203 return;
204
205 slow_path:
206 ltt_write_event_header_slow(config, ctx, event_id);
207 }
208
209 static
210 void ltt_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config,
211 struct lttng_ust_lib_ring_buffer_ctx *ctx,
212 uint32_t event_id)
213 {
214 struct ltt_channel *ltt_chan = channel_get_private(ctx->chan);
215 struct ltt_event *event = ctx->priv;
216
217 switch (ltt_chan->header_type) {
218 case 1: /* compact */
219 if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTT_RFLAG_EXTENDED))) {
220 uint32_t id_time = 0;
221
222 bt_bitfield_write(&id_time, uint32_t, 0, 5, event_id);
223 bt_bitfield_write(&id_time, uint32_t, 5, 27, ctx->tsc);
224 lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time));
225 } else {
226 uint8_t id = 0;
227 uint64_t timestamp = ctx->tsc;
228
229 bt_bitfield_write(&id, uint8_t, 0, 5, 31);
230 lib_ring_buffer_write(config, ctx, &id, sizeof(id));
231 /* Align extended struct on largest member */
232 lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t));
233 lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id));
234 lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t));
235 lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
236 }
237 break;
238 case 2: /* large */
239 {
240 if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTT_RFLAG_EXTENDED))) {
241 uint32_t timestamp = (uint32_t) ctx->tsc;
242 uint16_t id = event_id;
243
244 lib_ring_buffer_write(config, ctx, &id, sizeof(id));
245 lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint32_t));
246 lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
247 } else {
248 uint16_t id = 65535;
249 uint64_t timestamp = ctx->tsc;
250
251 lib_ring_buffer_write(config, ctx, &id, sizeof(id));
252 /* Align extended struct on largest member */
253 lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t));
254 lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id));
255 lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t));
256 lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
257 }
258 break;
259 }
260 default:
261 WARN_ON_ONCE(1);
262 }
263 ctx_record(ctx, ltt_chan, ltt_chan->ctx);
264 ctx_record(ctx, ltt_chan, event->ctx);
265 lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
266 }
267
268 static const struct lttng_ust_lib_ring_buffer_config client_config;
269
270 static u64 client_ring_buffer_clock_read(struct channel *chan)
271 {
272 return lib_ring_buffer_clock_read(chan);
273 }
274
275 static
276 size_t client_record_header_size(const struct lttng_ust_lib_ring_buffer_config *config,
277 struct channel *chan, size_t offset,
278 size_t *pre_header_padding,
279 struct lttng_ust_lib_ring_buffer_ctx *ctx)
280 {
281 return record_header_size(config, chan, offset,
282 pre_header_padding, ctx);
283 }
284
285 /**
286 * client_packet_header_size - called on buffer-switch to a new sub-buffer
287 *
288 * Return header size without padding after the structure. Don't use packed
289 * structure because gcc generates inefficient code on some architectures
290 * (powerpc, mips..)
291 */
292 static size_t client_packet_header_size(void)
293 {
294 return offsetof(struct packet_header, ctx.header_end);
295 }
296
297 static void client_buffer_begin(struct lttng_ust_lib_ring_buffer *buf, u64 tsc,
298 unsigned int subbuf_idx,
299 struct lttng_ust_shm_handle *handle)
300 {
301 struct channel *chan = shmp(handle, buf->backend.chan);
302 struct packet_header *header =
303 (struct packet_header *)
304 lib_ring_buffer_offset_address(&buf->backend,
305 subbuf_idx * chan->backend.subbuf_size,
306 handle);
307 struct ltt_channel *ltt_chan = channel_get_private(chan);
308 struct ltt_session *session = ltt_chan->session;
309
310 header->magic = CTF_MAGIC_NUMBER;
311 memcpy(header->uuid, session->uuid, sizeof(session->uuid));
312 header->stream_id = ltt_chan->id;
313 header->ctx.timestamp_begin = tsc;
314 header->ctx.timestamp_end = 0;
315 header->ctx.events_discarded = 0;
316 header->ctx.content_size = 0xFFFFFFFF; /* for debugging */
317 header->ctx.packet_size = 0xFFFFFFFF;
318 header->ctx.cpu_id = buf->backend.cpu;
319 }
320
321 /*
322 * offset is assumed to never be 0 here : never deliver a completely empty
323 * subbuffer. data_size is between 1 and subbuf_size.
324 */
325 static void client_buffer_end(struct lttng_ust_lib_ring_buffer *buf, u64 tsc,
326 unsigned int subbuf_idx, unsigned long data_size,
327 struct lttng_ust_shm_handle *handle)
328 {
329 struct channel *chan = shmp(handle, buf->backend.chan);
330 struct packet_header *header =
331 (struct packet_header *)
332 lib_ring_buffer_offset_address(&buf->backend,
333 subbuf_idx * chan->backend.subbuf_size,
334 handle);
335 unsigned long records_lost = 0;
336
337 header->ctx.timestamp_end = tsc;
338 header->ctx.content_size = data_size * CHAR_BIT; /* in bits */
339 header->ctx.packet_size = PAGE_ALIGN(data_size) * CHAR_BIT; /* in bits */
340 /*
341 * We do not care about the records lost count, because the metadata
342 * channel waits and retry.
343 */
344 (void) lib_ring_buffer_get_records_lost_full(&client_config, buf);
345 records_lost += lib_ring_buffer_get_records_lost_wrap(&client_config, buf);
346 records_lost += lib_ring_buffer_get_records_lost_big(&client_config, buf);
347 header->ctx.events_discarded = records_lost;
348 }
349
350 static int client_buffer_create(struct lttng_ust_lib_ring_buffer *buf, void *priv,
351 int cpu, const char *name, struct lttng_ust_shm_handle *handle)
352 {
353 return 0;
354 }
355
356 static void client_buffer_finalize(struct lttng_ust_lib_ring_buffer *buf, void *priv, int cpu, struct lttng_ust_shm_handle *handle)
357 {
358 }
359
360 static const struct lttng_ust_lib_ring_buffer_config client_config = {
361 .cb.ring_buffer_clock_read = client_ring_buffer_clock_read,
362 .cb.record_header_size = client_record_header_size,
363 .cb.subbuffer_header_size = client_packet_header_size,
364 .cb.buffer_begin = client_buffer_begin,
365 .cb.buffer_end = client_buffer_end,
366 .cb.buffer_create = client_buffer_create,
367 .cb.buffer_finalize = client_buffer_finalize,
368
369 .tsc_bits = 32,
370 .alloc = RING_BUFFER_ALLOC_PER_CPU,
371 .sync = RING_BUFFER_SYNC_GLOBAL,
372 .mode = RING_BUFFER_MODE_TEMPLATE,
373 .backend = RING_BUFFER_PAGE,
374 .output = RING_BUFFER_MMAP,
375 .oops = RING_BUFFER_OOPS_CONSISTENCY,
376 .ipi = RING_BUFFER_NO_IPI_BARRIER,
377 .wakeup = RING_BUFFER_WAKEUP_BY_WRITER,
378 .client_type = LTTNG_CLIENT_TYPE,
379 };
380
381 const struct lttng_ust_lib_ring_buffer_client_cb *LTTNG_CLIENT_CALLBACKS = &client_config.cb;
382
383 static
384 struct ltt_channel *_channel_create(const char *name,
385 struct ltt_channel *ltt_chan, void *buf_addr,
386 size_t subbuf_size, size_t num_subbuf,
387 unsigned int switch_timer_interval,
388 unsigned int read_timer_interval,
389 int *shm_fd, int *wait_fd,
390 uint64_t *memory_map_size)
391 {
392 ltt_chan->handle = channel_create(&client_config, name, ltt_chan, buf_addr,
393 subbuf_size, num_subbuf, switch_timer_interval,
394 read_timer_interval, shm_fd, wait_fd,
395 memory_map_size);
396 if (!ltt_chan->handle)
397 return NULL;
398 ltt_chan->chan = shmp(ltt_chan->handle, ltt_chan->handle->chan);
399 return ltt_chan;
400 }
401
402 static
403 void ltt_channel_destroy(struct ltt_channel *ltt_chan)
404 {
405 channel_destroy(ltt_chan->chan, ltt_chan->handle, 0);
406 }
407
408 static
409 struct lttng_ust_lib_ring_buffer *ltt_buffer_read_open(struct channel *chan,
410 struct lttng_ust_shm_handle *handle,
411 int *shm_fd, int *wait_fd,
412 uint64_t *memory_map_size)
413 {
414 struct lttng_ust_lib_ring_buffer *buf;
415 int cpu;
416
417 for_each_channel_cpu(cpu, chan) {
418 buf = channel_get_ring_buffer(&client_config, chan,
419 cpu, handle, shm_fd, wait_fd,
420 memory_map_size);
421 if (!lib_ring_buffer_open_read(buf, handle, 0))
422 return buf;
423 }
424 return NULL;
425 }
426
427 static
428 void ltt_buffer_read_close(struct lttng_ust_lib_ring_buffer *buf,
429 struct lttng_ust_shm_handle *handle)
430 {
431 lib_ring_buffer_release_read(buf, handle, 0);
432 }
433
434 static
435 int ltt_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx,
436 uint32_t event_id)
437 {
438 struct ltt_channel *ltt_chan = channel_get_private(ctx->chan);
439 int ret, cpu;
440
441 cpu = lib_ring_buffer_get_cpu(&client_config);
442 if (cpu < 0)
443 return -EPERM;
444 ctx->cpu = cpu;
445
446 switch (ltt_chan->header_type) {
447 case 1: /* compact */
448 if (event_id > 30)
449 ctx->rflags |= LTT_RFLAG_EXTENDED;
450 break;
451 case 2: /* large */
452 if (event_id > 65534)
453 ctx->rflags |= LTT_RFLAG_EXTENDED;
454 break;
455 default:
456 WARN_ON_ONCE(1);
457 }
458
459 ret = lib_ring_buffer_reserve(&client_config, ctx);
460 if (ret)
461 goto put;
462 ltt_write_event_header(&client_config, ctx, event_id);
463 return 0;
464 put:
465 lib_ring_buffer_put_cpu(&client_config);
466 return ret;
467 }
468
469 static
470 void ltt_event_commit(struct lttng_ust_lib_ring_buffer_ctx *ctx)
471 {
472 lib_ring_buffer_commit(&client_config, ctx);
473 lib_ring_buffer_put_cpu(&client_config);
474 }
475
476 static
477 void ltt_event_write(struct lttng_ust_lib_ring_buffer_ctx *ctx, const void *src,
478 size_t len)
479 {
480 lib_ring_buffer_write(&client_config, ctx, src, len);
481 }
482
483 #if 0
484 static
485 wait_queue_head_t *ltt_get_reader_wait_queue(struct channel *chan)
486 {
487 return &chan->read_wait;
488 }
489
490 static
491 wait_queue_head_t *ltt_get_hp_wait_queue(struct channel *chan)
492 {
493 return &chan->hp_wait;
494 }
495 #endif //0
496
497 static
498 int ltt_is_finalized(struct channel *chan)
499 {
500 return lib_ring_buffer_channel_is_finalized(chan);
501 }
502
503 static
504 int ltt_is_disabled(struct channel *chan)
505 {
506 return lib_ring_buffer_channel_is_disabled(chan);
507 }
508
509 static
510 int ltt_flush_buffer(struct channel *chan, struct lttng_ust_shm_handle *handle)
511 {
512 struct lttng_ust_lib_ring_buffer *buf;
513 int cpu;
514
515 for_each_channel_cpu(cpu, chan) {
516 int shm_fd, wait_fd;
517 uint64_t memory_map_size;
518
519 buf = channel_get_ring_buffer(&client_config, chan,
520 cpu, handle, &shm_fd, &wait_fd,
521 &memory_map_size);
522 lib_ring_buffer_switch(&client_config, buf,
523 SWITCH_ACTIVE, handle);
524 }
525 return 0;
526 }
527
528 static struct ltt_transport ltt_relay_transport = {
529 .name = "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap",
530 .ops = {
531 .channel_create = _channel_create,
532 .channel_destroy = ltt_channel_destroy,
533 .buffer_read_open = ltt_buffer_read_open,
534 .buffer_read_close = ltt_buffer_read_close,
535 .event_reserve = ltt_event_reserve,
536 .event_commit = ltt_event_commit,
537 .event_write = ltt_event_write,
538 .packet_avail_size = NULL, /* Would be racy anyway */
539 //.get_reader_wait_queue = ltt_get_reader_wait_queue,
540 //.get_hp_wait_queue = ltt_get_hp_wait_queue,
541 .is_finalized = ltt_is_finalized,
542 .is_disabled = ltt_is_disabled,
543 .flush_buffer = ltt_flush_buffer,
544 },
545 };
546
547 void RING_BUFFER_MODE_TEMPLATE_INIT(void)
548 {
549 DBG("LTT : ltt ring buffer client init\n");
550 ltt_transport_register(&ltt_relay_transport);
551 }
552
553 void RING_BUFFER_MODE_TEMPLATE_EXIT(void)
554 {
555 DBG("LTT : ltt ring buffer client exit\n");
556 ltt_transport_unregister(&ltt_relay_transport);
557 }
This page took 0.041646 seconds and 5 git commands to generate.