From b53047135828b997a4eeb2cae7344cb169aa9c8c Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Tue, 14 Apr 2020 11:31:36 -0400 Subject: [PATCH] Move headers under include/ --- LICENSE | 8 +- Makefile | 4 +- .../instrumentation}/events/README | 0 .../instrumentation}/events/lttng-module/9p.h | 4 +- .../events/lttng-module/arch/x86/exceptions.h | 4 +- .../lttng-module/arch/x86/irq_vectors.h | 4 +- .../lttng-module/arch/x86/kvm/mmutrace.h | 4 +- .../events/lttng-module/arch/x86/kvm/trace.h | 4 +- .../events/lttng-module/block.h | 4 +- .../events/lttng-module/btrfs.h | 4 +- .../events/lttng-module/compaction.h | 4 +- .../events/lttng-module/ext4.h | 4 +- .../events/lttng-module/gpio.h | 4 +- .../events/lttng-module/i2c.h | 4 +- .../events/lttng-module/irq.h | 4 +- .../events/lttng-module/jbd2.h | 4 +- .../events/lttng-module/kmem.h | 4 +- .../events/lttng-module/kvm.h | 4 +- .../events/lttng-module/lock.h | 4 +- .../events/lttng-module/lttng-statedump.h | 4 +- .../events/lttng-module/lttng-test.h | 4 +- .../events/lttng-module/lttng.h | 4 +- .../events/lttng-module/mm_vmscan.h | 4 +- .../events/lttng-module/module.h | 4 +- .../events/lttng-module/napi.h | 4 +- .../events/lttng-module/net.h | 6 +- .../events/lttng-module/power.h | 4 +- .../events/lttng-module/preemptirq.h | 4 +- .../events/lttng-module/printk.h | 4 +- .../events/lttng-module/random.h | 4 +- .../events/lttng-module/rcu.h | 4 +- .../events/lttng-module/regmap.h | 4 +- .../events/lttng-module/regulator.h | 4 +- .../events/lttng-module/rpc.h | 4 +- .../events/lttng-module/rpm.h | 4 +- .../events/lttng-module/sched.h | 4 +- .../events/lttng-module/scsi.h | 4 +- .../events/lttng-module/signal.h | 4 +- .../events/lttng-module/skb.h | 4 +- .../events/lttng-module/sock.h | 4 +- .../events/lttng-module/timer.h | 4 +- .../events/lttng-module/udp.h | 4 +- .../events/lttng-module/v4l2.h | 4 +- .../events/lttng-module/workqueue.h | 4 +- .../events/lttng-module/writeback.h | 4 +- .../syscalls/3.0.34/powerpc-32-syscalls | 0 .../syscalls/3.1.0-rc6/x86-32-syscalls | 0 .../syscalls/3.10.0-rc7/x86-64-syscalls | 0 .../syscalls/3.13.0/mips-32-syscalls | 0 .../syscalls/3.18.0/mips-32-syscalls | 0 .../syscalls/3.4.25/arm-32-syscalls | 0 .../syscalls/3.5.0/mips-32-syscalls | 0 .../syscalls/3.5.0/mips-64-syscalls | 0 .../syscalls/4.4.0/arm-64-syscalls | 0 .../instrumentation}/syscalls/README | 0 .../headers/arm-32-syscalls_integers.h | 4 +- .../arm-32-syscalls_integers_override.h | 0 .../headers/arm-32-syscalls_pointers.h | 4 +- .../arm-32-syscalls_pointers_override.h | 0 .../headers/arm-64-syscalls_integers.h | 4 +- .../arm-64-syscalls_integers_override.h | 0 .../headers/arm-64-syscalls_pointers.h | 4 +- .../arm-64-syscalls_pointers_override.h | 0 .../headers/compat_syscalls_integers.h | 0 .../headers/compat_syscalls_pointers.h | 0 .../headers/mips-32-syscalls_integers.h | 4 +- .../mips-32-syscalls_integers_override.h | 0 .../headers/mips-32-syscalls_pointers.h | 4 +- .../mips-32-syscalls_pointers_override.h | 0 .../headers/mips-64-syscalls_integers.h | 4 +- .../mips-64-syscalls_integers_override.h | 0 .../headers/mips-64-syscalls_pointers.h | 4 +- .../mips-64-syscalls_pointers_override.h | 0 .../headers/powerpc-32-syscalls_integers.h | 4 +- .../powerpc-32-syscalls_integers_override.h | 0 .../headers/powerpc-32-syscalls_pointers.h | 4 +- .../powerpc-32-syscalls_pointers_override.h | 0 .../syscalls/headers/syscalls_integers.h | 0 .../headers/syscalls_integers_override.h | 0 .../syscalls/headers/syscalls_pointers.h | 0 .../headers/syscalls_pointers_override.h | 0 .../syscalls/headers/syscalls_unknown.h | 4 +- .../headers/x86-32-syscalls_integers.h | 4 +- .../x86-32-syscalls_integers_override.h | 0 .../headers/x86-32-syscalls_pointers.h | 4 +- .../x86-32-syscalls_pointers_override.h | 0 .../headers/x86-64-syscalls_integers.h | 4 +- .../x86-64-syscalls_integers_override.h | 0 .../headers/x86-64-syscalls_pointers.h | 4 +- .../x86-64-syscalls_pointers_override.h | 0 .../syscalls/lttng-get-syscall-inout.sh | 0 .../syscalls/lttng-syscalls-extract.sh | 0 .../lttng-syscalls-extractor/Makefile | 0 .../linux-link-trace-syscalls-as-data.patch | 0 .../lttng-syscalls-extractor.c | 0 .../lttng-syscalls-generate-headers.sh | 4 +- .../syscalls/table-syscall-inout.txt | 0 {lib => include/lttng}/align.h | 4 +- {lib => include/lttng}/bitfield.h | 2 +- {lib => include/lttng}/bug.h | 2 +- {probes => include/lttng}/define_trace.h | 2 +- .../lttng/filter-bytecode.h | 0 .../lttng/lttng-abi-old.h | 2 +- lttng-abi.h => include/lttng/lttng-abi.h | 0 lttng-clock.h => include/lttng/lttng-clock.h | 0 .../lttng/lttng-cpuhotplug.h | 0 .../lttng/lttng-endian.h | 0 .../lttng}/lttng-events-nowrite.h | 0 .../lttng}/lttng-events-reset.h | 0 .../lttng}/lttng-events-write.h | 0 .../lttng/lttng-events.h | 10 +- .../lttng/lttng-filter.h | 4 +- .../lttng/lttng-kernel-version.h | 0 {probes => include/lttng}/lttng-probe-user.h | 0 .../lttng/lttng-string-utils.h | 0 .../lttng/lttng-tp-mempool.h | 0 .../lttng}/lttng-tracepoint-event-impl.h | 56 +- .../lttng}/lttng-tracepoint-event.h | 0 .../lttng/lttng-tracepoint.h | 0 .../lttng/lttng-tracer-core.h | 2 +- .../lttng/lttng-tracer.h | 4 +- {probes => include/lttng}/lttng-types.h | 12 +- .../lttng}/lttng_prio_heap.h | 0 include/ringbuffer/api.h | 26 +- include/ringbuffer/backend.h | 465 ++++++++++++++- include/ringbuffer/backend_internal.h | 533 +++++++++++++++++- include/ringbuffer/backend_types.h | 102 +++- include/ringbuffer/config.h | 308 +++++++++- include/ringbuffer/frontend.h | 236 +++++++- include/ringbuffer/frontend_api.h | 360 +++++++++++- include/ringbuffer/frontend_internal.h | 336 ++++++++++- include/ringbuffer/frontend_types.h | 190 ++++++- include/ringbuffer/iterator.h | 69 ++- include/ringbuffer/nohz.h | 31 +- include/ringbuffer/vatomic.h | 86 ++- include/ringbuffer/vfs.h | 168 +++++- {wrapper => include/wrapper}/irqflags.h | 0 {wrapper => include/wrapper}/trace-clock.h | 4 +- {wrapper => include/wrapper}/user_namespace.h | 0 lib/Kbuild | 2 +- lib/prio_heap/lttng_prio_heap.c | 2 +- lib/ringbuffer/api.h | 24 - lib/ringbuffer/backend.h | 463 --------------- lib/ringbuffer/backend_internal.h | 531 ----------------- lib/ringbuffer/backend_types.h | 100 ---- lib/ringbuffer/config.h | 306 ---------- lib/ringbuffer/frontend.h | 234 -------- lib/ringbuffer/frontend_api.h | 358 ------------ lib/ringbuffer/frontend_internal.h | 334 ----------- lib/ringbuffer/frontend_types.h | 188 ------ lib/ringbuffer/iterator.h | 67 --- lib/ringbuffer/nohz.h | 29 - lib/ringbuffer/ring_buffer_backend.c | 6 +- lib/ringbuffer/ring_buffer_frontend.c | 10 +- lib/ringbuffer/ring_buffer_iterator.c | 2 +- lib/ringbuffer/ring_buffer_mmap.c | 6 +- lib/ringbuffer/ring_buffer_splice.c | 6 +- lib/ringbuffer/ring_buffer_vfs.c | 8 +- lib/ringbuffer/vatomic.h | 84 --- lib/ringbuffer/vfs.h | 166 ------ lttng-abi.c | 20 +- lttng-calibrate.c | 4 +- lttng-clock.c | 4 +- lttng-context-callstack.c | 10 +- lttng-context-cgroup-ns.c | 6 +- lttng-context-cpu-id.c | 6 +- lttng-context-egid.c | 6 +- lttng-context-euid.c | 6 +- lttng-context-gid.c | 6 +- lttng-context-hostname.c | 6 +- lttng-context-interruptible.c | 6 +- lttng-context-ipc-ns.c | 6 +- lttng-context-migratable.c | 6 +- lttng-context-mnt-ns.c | 6 +- lttng-context-need-reschedule.c | 6 +- lttng-context-net-ns.c | 6 +- lttng-context-nice.c | 6 +- lttng-context-perf-counters.c | 6 +- lttng-context-pid-ns.c | 6 +- lttng-context-pid.c | 6 +- lttng-context-ppid.c | 6 +- lttng-context-preemptible.c | 6 +- lttng-context-prio.c | 6 +- lttng-context-procname.c | 8 +- lttng-context-sgid.c | 6 +- lttng-context-suid.c | 6 +- lttng-context-tid.c | 6 +- lttng-context-uid.c | 6 +- lttng-context-user-ns.c | 6 +- lttng-context-uts-ns.c | 6 +- lttng-context-vegid.c | 6 +- lttng-context-veuid.c | 6 +- lttng-context-vgid.c | 6 +- lttng-context-vpid.c | 6 +- lttng-context-vppid.c | 6 +- lttng-context-vsgid.c | 6 +- lttng-context-vsuid.c | 6 +- lttng-context-vtid.c | 6 +- lttng-context-vuid.c | 6 +- lttng-context.c | 4 +- lttng-events.c | 18 +- lttng-filter-interpreter.c | 4 +- lttng-filter-specialize.c | 4 +- lttng-filter-validator.c | 2 +- lttng-filter.c | 2 +- lttng-probes.c | 2 +- lttng-ring-buffer-client-discard.c | 2 +- lttng-ring-buffer-client-mmap-discard.c | 2 +- lttng-ring-buffer-client-mmap-overwrite.c | 2 +- lttng-ring-buffer-client-overwrite.c | 2 +- lttng-ring-buffer-client.h | 10 +- lttng-ring-buffer-metadata-client.c | 2 +- lttng-ring-buffer-metadata-client.h | 6 +- lttng-ring-buffer-metadata-mmap-client.c | 2 +- lttng-statedump-impl.c | 4 +- lttng-string-utils.c | 2 +- lttng-syscalls.c | 6 +- lttng-tp-mempool.c | 2 +- lttng-tracepoint.c | 2 +- lttng-tracker-id.c | 2 +- probes/Kbuild | 2 +- probes/lttng-kprobes.c | 6 +- probes/lttng-kretprobes.c | 6 +- probes/lttng-probe-9p.c | 2 +- probes/lttng-probe-block.c | 6 +- probes/lttng-probe-btrfs.c | 2 +- probes/lttng-probe-compaction.c | 2 +- probes/lttng-probe-ext4.c | 6 +- probes/lttng-probe-gpio.c | 2 +- probes/lttng-probe-i2c.c | 2 +- probes/lttng-probe-irq.c | 4 +- probes/lttng-probe-jbd2.c | 4 +- probes/lttng-probe-kmem.c | 4 +- probes/lttng-probe-kvm-x86-mmu.c | 4 +- probes/lttng-probe-kvm-x86.c | 4 +- probes/lttng-probe-kvm.c | 4 +- probes/lttng-probe-lock.c | 4 +- probes/lttng-probe-module.c | 4 +- probes/lttng-probe-napi.c | 4 +- probes/lttng-probe-net.c | 2 +- probes/lttng-probe-power.c | 4 +- probes/lttng-probe-preemptirq.c | 2 +- probes/lttng-probe-printk.c | 2 +- probes/lttng-probe-random.c | 2 +- probes/lttng-probe-rcu.c | 2 +- probes/lttng-probe-regmap.c | 4 +- probes/lttng-probe-regulator.c | 2 +- probes/lttng-probe-rpm.c | 2 +- probes/lttng-probe-sched.c | 4 +- probes/lttng-probe-scsi.c | 2 +- probes/lttng-probe-signal.c | 2 +- probes/lttng-probe-skb.c | 4 +- probes/lttng-probe-sock.c | 2 +- probes/lttng-probe-statedump.c | 4 +- probes/lttng-probe-sunrpc.c | 2 +- probes/lttng-probe-timer.c | 4 +- probes/lttng-probe-udp.c | 2 +- probes/lttng-probe-user.c | 2 +- probes/lttng-probe-v4l2.c | 2 +- probes/lttng-probe-vmscan.c | 4 +- probes/lttng-probe-workqueue.c | 4 +- probes/lttng-probe-writeback.c | 4 +- probes/lttng-probe-x86-exceptions.c | 4 +- probes/lttng-probe-x86-irq-vectors.c | 4 +- probes/lttng-uprobes.c | 6 +- probes/lttng.c | 2 +- probes/lttng.h | 14 - tests/Kbuild | 2 +- tests/clock-plugin/lttng-clock-plugin-test.c | 4 +- tests/probes/lttng-test.c | 4 +- wrapper/trace-clock.c => trace-clock.c | 0 271 files changed, 3329 insertions(+), 3365 deletions(-) rename {instrumentation => include/instrumentation}/events/README (100%) rename {instrumentation => include/instrumentation}/events/lttng-module/9p.h (93%) rename {instrumentation => include/instrumentation}/events/lttng-module/arch/x86/exceptions.h (95%) rename {instrumentation => include/instrumentation}/events/lttng-module/arch/x86/irq_vectors.h (97%) rename {instrumentation => include/instrumentation}/events/lttng-module/arch/x86/kvm/mmutrace.h (98%) rename {instrumentation => include/instrumentation}/events/lttng-module/arch/x86/kvm/trace.h (99%) rename {instrumentation => include/instrumentation}/events/lttng-module/block.h (99%) rename {instrumentation => include/instrumentation}/events/lttng-module/btrfs.h (99%) rename {instrumentation => include/instrumentation}/events/lttng-module/compaction.h (96%) rename {instrumentation => include/instrumentation}/events/lttng-module/ext4.h (99%) rename {instrumentation => include/instrumentation}/events/lttng-module/gpio.h (90%) rename {instrumentation => include/instrumentation}/events/lttng-module/i2c.h (96%) rename {instrumentation => include/instrumentation}/events/lttng-module/irq.h (97%) rename {instrumentation => include/instrumentation}/events/lttng-module/jbd2.h (98%) rename {instrumentation => include/instrumentation}/events/lttng-module/kmem.h (98%) rename {instrumentation => include/instrumentation}/events/lttng-module/kvm.h (98%) rename {instrumentation => include/instrumentation}/events/lttng-module/lock.h (94%) rename {instrumentation => include/instrumentation}/events/lttng-module/lttng-statedump.h (98%) rename {instrumentation => include/instrumentation}/events/lttng-module/lttng-test.h (96%) rename {instrumentation => include/instrumentation}/events/lttng-module/lttng.h (85%) rename {instrumentation => include/instrumentation}/events/lttng-module/mm_vmscan.h (98%) rename {instrumentation => include/instrumentation}/events/lttng-module/module.h (96%) rename {instrumentation => include/instrumentation}/events/lttng-module/napi.h (90%) rename {instrumentation => include/instrumentation}/events/lttng-module/net.h (99%) rename {instrumentation => include/instrumentation}/events/lttng-module/power.h (97%) rename {instrumentation => include/instrumentation}/events/lttng-module/preemptirq.h (95%) rename {instrumentation => include/instrumentation}/events/lttng-module/printk.h (85%) rename {instrumentation => include/instrumentation}/events/lttng-module/random.h (97%) rename {instrumentation => include/instrumentation}/events/lttng-module/rcu.h (99%) rename {instrumentation => include/instrumentation}/events/lttng-module/regmap.h (97%) rename {instrumentation => include/instrumentation}/events/lttng-module/regulator.h (96%) rename {instrumentation => include/instrumentation}/events/lttng-module/rpc.h (97%) rename {instrumentation => include/instrumentation}/events/lttng-module/rpm.h (95%) rename {instrumentation => include/instrumentation}/events/lttng-module/sched.h (99%) rename {instrumentation => include/instrumentation}/events/lttng-module/scsi.h (99%) rename {instrumentation => include/instrumentation}/events/lttng-module/signal.h (97%) rename {instrumentation => include/instrumentation}/events/lttng-module/skb.h (93%) rename {instrumentation => include/instrumentation}/events/lttng-module/sock.h (94%) rename {instrumentation => include/instrumentation}/events/lttng-module/timer.h (98%) rename {instrumentation => include/instrumentation}/events/lttng-module/udp.h (86%) rename {instrumentation => include/instrumentation}/events/lttng-module/v4l2.h (94%) rename {instrumentation => include/instrumentation}/events/lttng-module/workqueue.h (96%) rename {instrumentation => include/instrumentation}/events/lttng-module/writeback.h (99%) rename {instrumentation => include/instrumentation}/syscalls/3.0.34/powerpc-32-syscalls (100%) rename {instrumentation => include/instrumentation}/syscalls/3.1.0-rc6/x86-32-syscalls (100%) rename {instrumentation => include/instrumentation}/syscalls/3.10.0-rc7/x86-64-syscalls (100%) rename {instrumentation => include/instrumentation}/syscalls/3.13.0/mips-32-syscalls (100%) rename {instrumentation => include/instrumentation}/syscalls/3.18.0/mips-32-syscalls (100%) rename {instrumentation => include/instrumentation}/syscalls/3.4.25/arm-32-syscalls (100%) rename {instrumentation => include/instrumentation}/syscalls/3.5.0/mips-32-syscalls (100%) rename {instrumentation => include/instrumentation}/syscalls/3.5.0/mips-64-syscalls (100%) rename {instrumentation => include/instrumentation}/syscalls/4.4.0/arm-64-syscalls (100%) rename {instrumentation => include/instrumentation}/syscalls/README (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/arm-32-syscalls_integers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/arm-32-syscalls_integers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/arm-32-syscalls_pointers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/arm-32-syscalls_pointers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/arm-64-syscalls_integers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/arm-64-syscalls_integers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/arm-64-syscalls_pointers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/arm-64-syscalls_pointers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/compat_syscalls_integers.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/compat_syscalls_pointers.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/mips-32-syscalls_integers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/mips-32-syscalls_integers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/mips-32-syscalls_pointers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/mips-32-syscalls_pointers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/mips-64-syscalls_integers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/mips-64-syscalls_integers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/mips-64-syscalls_pointers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/mips-64-syscalls_pointers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/powerpc-32-syscalls_integers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/powerpc-32-syscalls_integers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/powerpc-32-syscalls_pointers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/powerpc-32-syscalls_pointers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/syscalls_integers.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/syscalls_integers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/syscalls_pointers.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/syscalls_pointers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/syscalls_unknown.h (94%) rename {instrumentation => include/instrumentation}/syscalls/headers/x86-32-syscalls_integers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/x86-32-syscalls_integers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/x86-32-syscalls_pointers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/x86-32-syscalls_pointers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/x86-64-syscalls_integers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/x86-64-syscalls_integers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/headers/x86-64-syscalls_pointers.h (99%) rename {instrumentation => include/instrumentation}/syscalls/headers/x86-64-syscalls_pointers_override.h (100%) rename {instrumentation => include/instrumentation}/syscalls/lttng-get-syscall-inout.sh (100%) rename {instrumentation => include/instrumentation}/syscalls/lttng-syscalls-extract.sh (100%) rename {instrumentation => include/instrumentation}/syscalls/lttng-syscalls-extractor/Makefile (100%) rename {instrumentation => include/instrumentation}/syscalls/lttng-syscalls-extractor/linux-link-trace-syscalls-as-data.patch (100%) rename {instrumentation => include/instrumentation}/syscalls/lttng-syscalls-extractor/lttng-syscalls-extractor.c (100%) rename {instrumentation => include/instrumentation}/syscalls/lttng-syscalls-generate-headers.sh (99%) rename {instrumentation => include/instrumentation}/syscalls/table-syscall-inout.txt (100%) rename {lib => include/lttng}/align.h (97%) rename {lib => include/lttng}/bitfield.h (99%) rename {lib => include/lttng}/bug.h (98%) rename {probes => include/lttng}/define_trace.h (98%) rename filter-bytecode.h => include/lttng/filter-bytecode.h (100%) rename lttng-abi-old.h => include/lttng/lttng-abi-old.h (99%) rename lttng-abi.h => include/lttng/lttng-abi.h (100%) rename lttng-clock.h => include/lttng/lttng-clock.h (100%) rename lttng-cpuhotplug.h => include/lttng/lttng-cpuhotplug.h (100%) rename lttng-endian.h => include/lttng/lttng-endian.h (100%) rename {probes => include/lttng}/lttng-events-nowrite.h (100%) rename {probes => include/lttng}/lttng-events-reset.h (100%) rename {probes => include/lttng}/lttng-events-write.h (100%) rename lttng-events.h => include/lttng/lttng-events.h (99%) rename lttng-filter.h => include/lttng/lttng-filter.h (98%) rename lttng-kernel-version.h => include/lttng/lttng-kernel-version.h (100%) rename {probes => include/lttng}/lttng-probe-user.h (100%) rename lttng-string-utils.h => include/lttng/lttng-string-utils.h (100%) rename lttng-tp-mempool.h => include/lttng/lttng-tp-mempool.h (100%) rename {probes => include/lttng}/lttng-tracepoint-event-impl.h (97%) rename {probes => include/lttng}/lttng-tracepoint-event.h (100%) rename lttng-tracepoint.h => include/lttng/lttng-tracepoint.h (100%) rename lttng-tracer-core.h => include/lttng/lttng-tracer-core.h (93%) rename lttng-tracer.h => include/lttng/lttng-tracer.h (96%) rename {probes => include/lttng}/lttng-types.h (90%) rename {lib/prio_heap => include/lttng}/lttng_prio_heap.h (100%) rename {wrapper => include/wrapper}/irqflags.h (100%) rename {wrapper => include/wrapper}/trace-clock.h (98%) rename {wrapper => include/wrapper}/user_namespace.h (100%) delete mode 100644 lib/ringbuffer/api.h delete mode 100644 lib/ringbuffer/backend.h delete mode 100644 lib/ringbuffer/backend_internal.h delete mode 100644 lib/ringbuffer/backend_types.h delete mode 100644 lib/ringbuffer/config.h delete mode 100644 lib/ringbuffer/frontend.h delete mode 100644 lib/ringbuffer/frontend_api.h delete mode 100644 lib/ringbuffer/frontend_internal.h delete mode 100644 lib/ringbuffer/frontend_types.h delete mode 100644 lib/ringbuffer/iterator.h delete mode 100644 lib/ringbuffer/nohz.h delete mode 100644 lib/ringbuffer/vatomic.h delete mode 100644 lib/ringbuffer/vfs.h delete mode 100644 probes/lttng.h rename wrapper/trace-clock.c => trace-clock.c (100%) diff --git a/LICENSE b/LICENSE index 060e266a..3b8eaf84 100644 --- a/LICENSE +++ b/LICENSE @@ -22,12 +22,12 @@ instrumentation/events/lttng-module/*.h These files are licensed under an MIT-style license. See LICENSES/MIT for details. -lib/prio_heap/lttng_prio_heap.h +include/lttng/lttng_prio_heap.h +include/lttng/bitfield.h +include/lttng/filter-bytecode.h +include/lttng/lttng-filter.h lib/prio_heap/lttng_prio_heap.c -lib/bitfield.h -filter-bytecode.h lttng-filter-interpreter.c lttng-filter-specialize.c lttng-filter-validator.c lttng-filter.c -lttng-filter.h diff --git a/Makefile b/Makefile index 3486002b..9de6aed1 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ ifneq ($(KERNELRELEASE),) include $(TOP_LTTNG_MODULES_DIR)/Kbuild.common - ccflags-y += -I$(TOP_LTTNG_MODULES_DIR) + ccflags-y += -I$(TOP_LTTNG_MODULES_DIR)/include obj-$(CONFIG_LTTNG) += lttng-ring-buffer-client-discard.o obj-$(CONFIG_LTTNG) += lttng-ring-buffer-client-overwrite.o @@ -64,7 +64,7 @@ ifneq ($(KERNELRELEASE),) lttng-context-need-reschedule.o \ lttng-context-callstack.o lttng-calibrate.o \ lttng-context-hostname.o \ - probes/lttng.o wrapper/trace-clock.o \ + probes/lttng.o trace-clock.o \ lttng-tracker-id.o \ lttng-filter.o lttng-filter-interpreter.o \ lttng-filter-specialize.o \ diff --git a/instrumentation/events/README b/include/instrumentation/events/README similarity index 100% rename from instrumentation/events/README rename to include/instrumentation/events/README diff --git a/instrumentation/events/lttng-module/9p.h b/include/instrumentation/events/lttng-module/9p.h similarity index 93% rename from instrumentation/events/lttng-module/9p.h rename to include/instrumentation/events/lttng-module/9p.h index aa7da309..43283e8a 100644 --- a/instrumentation/events/lttng-module/9p.h +++ b/include/instrumentation/events/lttng-module/9p.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_9P_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_9P_H -#include +#include LTTNG_TRACEPOINT_EVENT(9p_client_req, @@ -51,4 +51,4 @@ LTTNG_TRACEPOINT_EVENT(9p_protocol_dump, #endif /* LTTNG_TRACE_9P_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/arch/x86/exceptions.h b/include/instrumentation/events/lttng-module/arch/x86/exceptions.h similarity index 95% rename from instrumentation/events/lttng-module/arch/x86/exceptions.h rename to include/instrumentation/events/lttng-module/arch/x86/exceptions.h index ec34d35b..d8bca0d9 100644 --- a/instrumentation/events/lttng-module/arch/x86/exceptions.h +++ b/include/instrumentation/events/lttng-module/arch/x86/exceptions.h @@ -2,7 +2,7 @@ #if !defined(LTTNG_TRACE_EXCEPTIONS_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_EXCEPTIONS_H -#include +#include #include #undef TRACE_SYSTEM @@ -50,4 +50,4 @@ LTTNG_EXCEPTIONS_TRACEPOINT_EVENT_INSTANCE_MAP(page_fault_kernel, #define TRACE_INCLUDE_FILE exceptions /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/arch/x86/irq_vectors.h b/include/instrumentation/events/lttng-module/arch/x86/irq_vectors.h similarity index 97% rename from instrumentation/events/lttng-module/arch/x86/irq_vectors.h rename to include/instrumentation/events/lttng-module/arch/x86/irq_vectors.h index aa0d6d71..bcf4819a 100644 --- a/instrumentation/events/lttng-module/arch/x86/irq_vectors.h +++ b/include/instrumentation/events/lttng-module/arch/x86/irq_vectors.h @@ -2,7 +2,7 @@ #if !defined(LTTNG_TRACE_IRQ_VECTORS_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_IRQ_VECTORS_H -#include +#include #include #undef TRACE_SYSTEM @@ -113,4 +113,4 @@ LTTNG_IRQ_VECTOR_TRACEPOINT_EVENT_INSTANCE_MAP(thermal_apic, #define TRACE_INCLUDE_FILE irq_vectors /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/arch/x86/kvm/mmutrace.h b/include/instrumentation/events/lttng-module/arch/x86/kvm/mmutrace.h similarity index 98% rename from instrumentation/events/lttng-module/arch/x86/kvm/mmutrace.h rename to include/instrumentation/events/lttng-module/arch/x86/kvm/mmutrace.h index 9f210284..70d6e865 100644 --- a/instrumentation/events/lttng-module/arch/x86/kvm/mmutrace.h +++ b/include/instrumentation/events/lttng-module/arch/x86/kvm/mmutrace.h @@ -2,7 +2,7 @@ #if !defined(LTTNG_TRACE_KVM_MMU_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_KVM_MMU_H -#include +#include #include @@ -177,4 +177,4 @@ LTTNG_TRACEPOINT_EVENT_MAP( #define TRACE_INCLUDE_FILE mmutrace /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/arch/x86/kvm/trace.h b/include/instrumentation/events/lttng-module/arch/x86/kvm/trace.h similarity index 99% rename from instrumentation/events/lttng-module/arch/x86/kvm/trace.h rename to include/instrumentation/events/lttng-module/arch/x86/kvm/trace.h index 19b1c086..22a1a580 100644 --- a/instrumentation/events/lttng-module/arch/x86/kvm/trace.h +++ b/include/instrumentation/events/lttng-module/arch/x86/kvm/trace.h @@ -2,7 +2,7 @@ #if !defined(LTTNG_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_KVM_H -#include +#include #include #include #include @@ -500,4 +500,4 @@ LTTNG_TRACEPOINT_EVENT_MAP(kvm_track_tsc, kvm_x86_track_tsc, #define TRACE_INCLUDE_FILE trace /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/block.h b/include/instrumentation/events/lttng-module/block.h similarity index 99% rename from instrumentation/events/lttng-module/block.h rename to include/instrumentation/events/lttng-module/block.h index 3ca51369..b4fd4bfc 100644 --- a/instrumentation/events/lttng-module/block.h +++ b/include/instrumentation/events/lttng-module/block.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_BLOCK_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_BLOCK_H -#include +#include #include #include #include @@ -535,4 +535,4 @@ LTTNG_TRACEPOINT_EVENT(block_rq_remap, #endif /* LTTNG_TRACE_BLOCK_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/btrfs.h b/include/instrumentation/events/lttng-module/btrfs.h similarity index 99% rename from instrumentation/events/lttng-module/btrfs.h rename to include/instrumentation/events/lttng-module/btrfs.h index f4fdebca..e7df77cc 100644 --- a/instrumentation/events/lttng-module/btrfs.h +++ b/include/instrumentation/events/lttng-module/btrfs.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_BTRFS_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_BTRFS_H -#include +#include #include #ifndef _TRACE_BTRFS_DEF_ @@ -633,4 +633,4 @@ LTTNG_TRACEPOINT_EVENT_MAP(free_extent_state, #endif /* LTTNG_TRACE_BTRFS_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/compaction.h b/include/instrumentation/events/lttng-module/compaction.h similarity index 96% rename from instrumentation/events/lttng-module/compaction.h rename to include/instrumentation/events/lttng-module/compaction.h index 2fed7290..1e5cccfc 100644 --- a/instrumentation/events/lttng-module/compaction.h +++ b/include/instrumentation/events/lttng-module/compaction.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_COMPACTION_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_COMPACTION_H -#include +#include #include LTTNG_TRACEPOINT_EVENT_CLASS(compaction_isolate_template, @@ -92,4 +92,4 @@ LTTNG_TRACEPOINT_EVENT_CODE_MAP(mm_compaction_migratepages, #endif /* LTTNG_TRACE_COMPACTION_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/ext4.h b/include/instrumentation/events/lttng-module/ext4.h similarity index 99% rename from instrumentation/events/lttng-module/ext4.h rename to include/instrumentation/events/lttng-module/ext4.h index b9586a75..26118a9e 100644 --- a/instrumentation/events/lttng-module/ext4.h +++ b/include/instrumentation/events/lttng-module/ext4.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_EXT4_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_EXT4_H -#include +#include #include #ifndef _TRACE_EXT4_DEF_ @@ -1208,4 +1208,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(ext4__es_extent, ext4_es_cache_extent, #endif /* LTTNG_TRACE_EXT4_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/gpio.h b/include/instrumentation/events/lttng-module/gpio.h similarity index 90% rename from instrumentation/events/lttng-module/gpio.h rename to include/instrumentation/events/lttng-module/gpio.h index 7a9246e7..c0419d1e 100644 --- a/instrumentation/events/lttng-module/gpio.h +++ b/include/instrumentation/events/lttng-module/gpio.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_GPIO_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_GPIO_H -#include +#include LTTNG_TRACEPOINT_EVENT(gpio_direction, @@ -36,4 +36,4 @@ LTTNG_TRACEPOINT_EVENT(gpio_value, #endif /* if !defined(LTTNG_TRACE_GPIO_H) || defined(TRACE_HEADER_MULTI_READ) */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/i2c.h b/include/instrumentation/events/lttng-module/i2c.h similarity index 96% rename from instrumentation/events/lttng-module/i2c.h rename to include/instrumentation/events/lttng-module/i2c.h index 7fad2f7d..f65c2849 100644 --- a/instrumentation/events/lttng-module/i2c.h +++ b/include/instrumentation/events/lttng-module/i2c.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_I2C_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_I2C_H -#include +#include /* * __i2c_transfer() write request @@ -117,4 +117,4 @@ LTTNG_TRACEPOINT_EVENT(i2c_result, #endif /* LTTNG_TRACE_I2C_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/irq.h b/include/instrumentation/events/lttng-module/irq.h similarity index 97% rename from instrumentation/events/lttng-module/irq.h rename to include/instrumentation/events/lttng-module/irq.h index e1bbbafe..3995add4 100644 --- a/instrumentation/events/lttng-module/irq.h +++ b/include/instrumentation/events/lttng-module/irq.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_IRQ_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_IRQ_H -#include +#include #ifndef _TRACE_IRQ_DEF_ #define _TRACE_IRQ_DEF_ @@ -123,4 +123,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(irq_softirq, softirq_raise, #endif /* LTTNG_TRACE_IRQ_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/jbd2.h b/include/instrumentation/events/lttng-module/jbd2.h similarity index 98% rename from instrumentation/events/lttng-module/jbd2.h rename to include/instrumentation/events/lttng-module/jbd2.h index db2f0d5f..1b0d00d3 100644 --- a/instrumentation/events/lttng-module/jbd2.h +++ b/include/instrumentation/events/lttng-module/jbd2.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_JBD2_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_JBD2_H -#include +#include #include #ifndef _TRACE_JBD2_DEF @@ -165,4 +165,4 @@ LTTNG_TRACEPOINT_EVENT(jbd2_write_superblock, #endif /* LTTNG_TRACE_JBD2_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/kmem.h b/include/instrumentation/events/lttng-module/kmem.h similarity index 98% rename from instrumentation/events/lttng-module/kmem.h rename to include/instrumentation/events/lttng-module/kmem.h index 479f0fef..560b9be7 100644 --- a/instrumentation/events/lttng-module/kmem.h +++ b/include/instrumentation/events/lttng-module/kmem.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_KMEM_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_KMEM_H -#include +#include #include LTTNG_TRACEPOINT_EVENT_CLASS(kmem_alloc, @@ -216,4 +216,4 @@ LTTNG_TRACEPOINT_EVENT_MAP(mm_page_alloc_extfrag, #endif /* LTTNG_TRACE_KMEM_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/kvm.h b/include/instrumentation/events/lttng-module/kvm.h similarity index 98% rename from instrumentation/events/lttng-module/kvm.h rename to include/instrumentation/events/lttng-module/kvm.h index cdcc35b2..35b65839 100644 --- a/instrumentation/events/lttng-module/kvm.h +++ b/include/instrumentation/events/lttng-module/kvm.h @@ -2,7 +2,7 @@ #if !defined(LTTNG_TRACE_KVM_MAIN_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_KVM_MAIN_H -#include +#include #undef TRACE_SYSTEM #define TRACE_SYSTEM kvm @@ -182,4 +182,4 @@ LTTNG_TRACEPOINT_EVENT( #endif /* LTTNG_TRACE_KVM_MAIN_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/lock.h b/include/instrumentation/events/lttng-module/lock.h similarity index 94% rename from instrumentation/events/lttng-module/lock.h rename to include/instrumentation/events/lttng-module/lock.h index f384fd4a..5f0f09da 100644 --- a/instrumentation/events/lttng-module/lock.h +++ b/include/instrumentation/events/lttng-module/lock.h @@ -7,7 +7,7 @@ #define LTTNG_TRACE_LOCK_H #include -#include +#include #ifdef CONFIG_LOCKDEP @@ -68,4 +68,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(lock, lock_acquired, #endif /* LTTNG_TRACE_LOCK_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/lttng-statedump.h b/include/instrumentation/events/lttng-module/lttng-statedump.h similarity index 98% rename from instrumentation/events/lttng-module/lttng-statedump.h rename to include/instrumentation/events/lttng-module/lttng-statedump.h index 7ca72ccd..ee3a841e 100644 --- a/instrumentation/events/lttng-module/lttng-statedump.h +++ b/include/instrumentation/events/lttng-module/lttng-statedump.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_LTTNG_STATEDUMP_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_LTTNG_STATEDUMP_H -#include +#include #include #include #include @@ -250,4 +250,4 @@ LTTNG_TRACEPOINT_EVENT(lttng_statedump_cpu_topology, #endif /* LTTNG_TRACE_LTTNG_STATEDUMP_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/lttng-test.h b/include/instrumentation/events/lttng-module/lttng-test.h similarity index 96% rename from instrumentation/events/lttng-module/lttng-test.h rename to include/instrumentation/events/lttng-module/lttng-test.h index 761fcf08..ad4b1779 100644 --- a/instrumentation/events/lttng-module/lttng-test.h +++ b/include/instrumentation/events/lttng-module/lttng-test.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_LTTNG_TEST_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_LTTNG_TEST_H -#include +#include #include LTTNG_TRACEPOINT_ENUM( @@ -54,4 +54,4 @@ LTTNG_TRACEPOINT_EVENT(lttng_test_filter_event, #endif /* LTTNG_TRACE_LTTNG_TEST_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/lttng.h b/include/instrumentation/events/lttng-module/lttng.h similarity index 85% rename from instrumentation/events/lttng-module/lttng.h rename to include/instrumentation/events/lttng-module/lttng.h index 376fd537..50a3d693 100644 --- a/instrumentation/events/lttng-module/lttng.h +++ b/include/instrumentation/events/lttng-module/lttng.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_LTTNG_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_LTTNG_H -#include +#include LTTNG_TRACEPOINT_EVENT(lttng_logger, TP_PROTO(const char __user *text, size_t len), @@ -18,4 +18,4 @@ LTTNG_TRACEPOINT_EVENT(lttng_logger, #endif /* LTTNG_TRACE_LTTNG_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/mm_vmscan.h b/include/instrumentation/events/lttng-module/mm_vmscan.h similarity index 98% rename from instrumentation/events/lttng-module/mm_vmscan.h rename to include/instrumentation/events/lttng-module/mm_vmscan.h index a589eb65..c0a259b4 100644 --- a/instrumentation/events/lttng-module/mm_vmscan.h +++ b/include/instrumentation/events/lttng-module/mm_vmscan.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_MM_VMSCAN_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_MM_VMSCAN_H -#include +#include #include #include #include @@ -244,4 +244,4 @@ LTTNG_TRACEPOINT_EVENT(mm_vmscan_lru_shrink_inactive, #endif /* LTTNG_TRACE_MM_VMSCAN_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/module.h b/include/instrumentation/events/lttng-module/module.h similarity index 96% rename from instrumentation/events/lttng-module/module.h rename to include/instrumentation/events/lttng-module/module.h index 5299e17f..04baeae0 100644 --- a/instrumentation/events/lttng-module/module.h +++ b/include/instrumentation/events/lttng-module/module.h @@ -15,7 +15,7 @@ #if !defined(LTTNG_TRACE_MODULE_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_MODULE_H -#include +#include #ifdef CONFIG_MODULES @@ -97,4 +97,4 @@ LTTNG_TRACEPOINT_EVENT(module_request, #endif /* LTTNG_TRACE_MODULE_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/napi.h b/include/instrumentation/events/lttng-module/napi.h similarity index 90% rename from instrumentation/events/lttng-module/napi.h rename to include/instrumentation/events/lttng-module/napi.h index 4bb76f7f..a7415206 100644 --- a/instrumentation/events/lttng-module/napi.h +++ b/include/instrumentation/events/lttng-module/napi.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_NAPI_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_NAPI_H -#include +#include #include #include @@ -30,4 +30,4 @@ LTTNG_TRACEPOINT_EVENT(napi_poll, #endif /* LTTNG_TRACE_NAPI_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/net.h b/include/instrumentation/events/lttng-module/net.h similarity index 99% rename from instrumentation/events/lttng-module/net.h rename to include/instrumentation/events/lttng-module/net.h index 40acebf8..4d0cbf1e 100644 --- a/instrumentation/events/lttng-module/net.h +++ b/include/instrumentation/events/lttng-module/net.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_NET_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_NET_H -#include +#include #include #include #include @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #ifndef ONCE_LTTNG_NET_H @@ -855,4 +855,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(net_dev_receive_exit_template, #endif /* LTTNG_TRACE_NET_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/power.h b/include/instrumentation/events/lttng-module/power.h similarity index 97% rename from instrumentation/events/lttng-module/power.h rename to include/instrumentation/events/lttng-module/power.h index e9f56dc9..299183cc 100644 --- a/instrumentation/events/lttng-module/power.h +++ b/include/instrumentation/events/lttng-module/power.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_POWER_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_POWER_H -#include +#include #include LTTNG_TRACEPOINT_EVENT_CLASS(power_cpu, @@ -158,4 +158,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(power_domain, power_domain_target, #endif /* LTTNG_TRACE_POWER_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/preemptirq.h b/include/instrumentation/events/lttng-module/preemptirq.h similarity index 95% rename from instrumentation/events/lttng-module/preemptirq.h rename to include/instrumentation/events/lttng-module/preemptirq.h index b69868ed..27960427 100644 --- a/instrumentation/events/lttng-module/preemptirq.h +++ b/include/instrumentation/events/lttng-module/preemptirq.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include /* * The preemptirq probe is built when CONFIG_PREEMPTIRQ_EVENTS is defined. @@ -79,6 +79,6 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(preemptirq_template, preempt_enable, #endif /* LTTNG_TRACE_PREEMPTIRQ_H */ /* This part must be outside protection */ -#include +#include #endif /* CONFIG_PREEMPTIRQ_EVENTS */ diff --git a/instrumentation/events/lttng-module/printk.h b/include/instrumentation/events/lttng-module/printk.h similarity index 85% rename from instrumentation/events/lttng-module/printk.h rename to include/instrumentation/events/lttng-module/printk.h index 26b04a95..3cbf2c9a 100644 --- a/instrumentation/events/lttng-module/printk.h +++ b/include/instrumentation/events/lttng-module/printk.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_PRINTK_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_PRINTK_H -#include +#include LTTNG_TRACEPOINT_EVENT_MAP(console, @@ -23,4 +23,4 @@ LTTNG_TRACEPOINT_EVENT_MAP(console, #endif /* LTTNG_TRACE_PRINTK_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/random.h b/include/instrumentation/events/lttng-module/random.h similarity index 97% rename from instrumentation/events/lttng-module/random.h rename to include/instrumentation/events/lttng-module/random.h index 3d980f90..633bf67d 100644 --- a/instrumentation/events/lttng-module/random.h +++ b/include/instrumentation/events/lttng-module/random.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_RANDOM_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_RANDOM_H -#include +#include #include LTTNG_TRACEPOINT_EVENT_CLASS(random__mix_pool_bytes, @@ -109,4 +109,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP(random__extract_entropy, extract_entropy_use #endif /* LTTNG_TRACE_RANDOM_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/rcu.h b/include/instrumentation/events/lttng-module/rcu.h similarity index 99% rename from instrumentation/events/lttng-module/rcu.h rename to include/instrumentation/events/lttng-module/rcu.h index be4a074c..73dc18c2 100644 --- a/instrumentation/events/lttng-module/rcu.h +++ b/include/instrumentation/events/lttng-module/rcu.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_RCU_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_RCU_H -#include +#include /* * Tracepoint for start/end markers used for utilization calculations. @@ -442,4 +442,4 @@ LTTNG_TRACEPOINT_EVENT(rcu_barrier, #endif /* LTTNG_TRACE_RCU_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/regmap.h b/include/instrumentation/events/lttng-module/regmap.h similarity index 97% rename from instrumentation/events/lttng-module/regmap.h rename to include/instrumentation/events/lttng-module/regmap.h index 1d7e954d..d7277cb2 100644 --- a/instrumentation/events/lttng-module/regmap.h +++ b/include/instrumentation/events/lttng-module/regmap.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_REGMAP_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_REGMAP_H -#include +#include #include #ifndef _TRACE_REGMAP_DEF_ @@ -141,4 +141,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(regmap_bool, regmap_cache_bypass, #endif /* LTTNG_TRACE_REGMAP_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/regulator.h b/include/instrumentation/events/lttng-module/regulator.h similarity index 96% rename from instrumentation/events/lttng-module/regulator.h rename to include/instrumentation/events/lttng-module/regulator.h index 46ad50d2..44f83caa 100644 --- a/instrumentation/events/lttng-module/regulator.h +++ b/include/instrumentation/events/lttng-module/regulator.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_REGULATOR_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_REGULATOR_H -#include +#include #include /* @@ -115,4 +115,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(regulator_value, regulator_set_voltage_complete, #endif /* _TRACE_POWER_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/rpc.h b/include/instrumentation/events/lttng-module/rpc.h similarity index 97% rename from instrumentation/events/lttng-module/rpc.h rename to include/instrumentation/events/lttng-module/rpc.h index 8b15cfab..a6bada96 100644 --- a/instrumentation/events/lttng-module/rpc.h +++ b/include/instrumentation/events/lttng-module/rpc.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_RPC_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_RPC_H -#include +#include #include #include @@ -131,4 +131,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(rpc_task_queued, rpc_task_wakeup, #endif /* LTTNG_TRACE_RPC_H */ -#include +#include diff --git a/instrumentation/events/lttng-module/rpm.h b/include/instrumentation/events/lttng-module/rpm.h similarity index 95% rename from instrumentation/events/lttng-module/rpm.h rename to include/instrumentation/events/lttng-module/rpm.h index 7aa44d79..71909fc8 100644 --- a/instrumentation/events/lttng-module/rpm.h +++ b/include/instrumentation/events/lttng-module/rpm.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_RUNTIME_POWER_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_RUNTIME_POWER_H -#include +#include #include #ifndef _TRACE_RPM_DEF_ @@ -70,4 +70,4 @@ LTTNG_TRACEPOINT_EVENT(rpm_return_int, #endif /* LTTNG_TRACE_RUNTIME_POWER_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/sched.h b/include/instrumentation/events/lttng-module/sched.h similarity index 99% rename from instrumentation/events/lttng-module/sched.h rename to include/instrumentation/events/lttng-module/sched.h index 20890ac0..a13e5990 100644 --- a/instrumentation/events/lttng-module/sched.h +++ b/include/instrumentation/events/lttng-module/sched.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_SCHED_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_SCHED_H -#include +#include #include #include #include @@ -406,4 +406,4 @@ LTTNG_TRACEPOINT_EVENT(sched_pi_setprio, #endif /* LTTNG_TRACE_SCHED_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/scsi.h b/include/instrumentation/events/lttng-module/scsi.h similarity index 99% rename from instrumentation/events/lttng-module/scsi.h rename to include/instrumentation/events/lttng-module/scsi.h index 0a864d1a..ea317adc 100644 --- a/instrumentation/events/lttng-module/scsi.h +++ b/include/instrumentation/events/lttng-module/scsi.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_SCSI_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_SCSI_H -#include +#include #include #include #include @@ -286,4 +286,4 @@ LTTNG_TRACEPOINT_EVENT(scsi_eh_wakeup, #endif /* LTTNG_TRACE_SCSI_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/signal.h b/include/instrumentation/events/lttng-module/signal.h similarity index 97% rename from instrumentation/events/lttng-module/signal.h rename to include/instrumentation/events/lttng-module/signal.h index 810dcbea..8af34d7a 100644 --- a/instrumentation/events/lttng-module/signal.h +++ b/include/instrumentation/events/lttng-module/signal.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_SIGNAL_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_SIGNAL_H -#include +#include #ifndef _TRACE_SIGNAL_DEF #define _TRACE_SIGNAL_DEF @@ -83,4 +83,4 @@ LTTNG_TRACEPOINT_EVENT(signal_deliver, #endif /* LTTNG_TRACE_SIGNAL_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/skb.h b/include/instrumentation/events/lttng-module/skb.h similarity index 93% rename from instrumentation/events/lttng-module/skb.h rename to include/instrumentation/events/lttng-module/skb.h index 5126b57d..63afe180 100644 --- a/instrumentation/events/lttng-module/skb.h +++ b/include/instrumentation/events/lttng-module/skb.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_SKB_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_SKB_H -#include +#include #include #include @@ -55,4 +55,4 @@ LTTNG_TRACEPOINT_EVENT(skb_copy_datagram_iovec, #endif /* LTTNG_TRACE_SKB_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/sock.h b/include/instrumentation/events/lttng-module/sock.h similarity index 94% rename from instrumentation/events/lttng-module/sock.h rename to include/instrumentation/events/lttng-module/sock.h index 43e12f3f..532f511e 100644 --- a/instrumentation/events/lttng-module/sock.h +++ b/include/instrumentation/events/lttng-module/sock.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_SOCK_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_SOCK_H -#include +#include #include LTTNG_TRACEPOINT_EVENT(sock_rcvqueue_full, @@ -43,4 +43,4 @@ LTTNG_TRACEPOINT_EVENT(sock_exceed_buf_limit, #endif /* LTTNG_TRACE_SOCK_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/timer.h b/include/instrumentation/events/lttng-module/timer.h similarity index 98% rename from instrumentation/events/lttng-module/timer.h rename to include/instrumentation/events/lttng-module/timer.h index 263f6213..c8f3507c 100644 --- a/instrumentation/events/lttng-module/timer.h +++ b/include/instrumentation/events/lttng-module/timer.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_TIMER_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_TIMER_H -#include +#include #ifndef _TRACE_TIMER_DEF_ #define _TRACE_TIMER_DEF_ @@ -268,4 +268,4 @@ LTTNG_TRACEPOINT_EVENT_MAP(itimer_expire, #endif /* LTTNG_TRACE_TIMER_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/udp.h b/include/instrumentation/events/lttng-module/udp.h similarity index 86% rename from instrumentation/events/lttng-module/udp.h rename to include/instrumentation/events/lttng-module/udp.h index 427fa006..ec7ba2fd 100644 --- a/instrumentation/events/lttng-module/udp.h +++ b/include/instrumentation/events/lttng-module/udp.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_UDP_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_UDP_H -#include +#include #include LTTNG_TRACEPOINT_EVENT(udp_fail_queue_rcv_skb, @@ -23,4 +23,4 @@ LTTNG_TRACEPOINT_EVENT(udp_fail_queue_rcv_skb, #endif /* LTTNG_TRACE_UDP_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/v4l2.h b/include/instrumentation/events/lttng-module/v4l2.h similarity index 94% rename from instrumentation/events/lttng-module/v4l2.h rename to include/instrumentation/events/lttng-module/v4l2.h index 5e5bae16..c42aa4c5 100644 --- a/instrumentation/events/lttng-module/v4l2.h +++ b/include/instrumentation/events/lttng-module/v4l2.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_V4L2_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_V4L2_H -#include +#include LTTNG_TRACEPOINT_EVENT_CLASS(v4l2_class, @@ -52,4 +52,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(v4l2_class, #endif /* if !defined(LTTNG_TRACE_V4L2_H) || defined(TRACE_HEADER_MULTI_READ) */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/workqueue.h b/include/instrumentation/events/lttng-module/workqueue.h similarity index 96% rename from instrumentation/events/lttng-module/workqueue.h rename to include/instrumentation/events/lttng-module/workqueue.h index c33dd3b1..dcd3f017 100644 --- a/instrumentation/events/lttng-module/workqueue.h +++ b/include/instrumentation/events/lttng-module/workqueue.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_WORKQUEUE_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_WORKQUEUE_H -#include +#include #include #ifndef _TRACE_WORKQUEUE_DEF_ @@ -106,4 +106,4 @@ LTTNG_TRACEPOINT_EVENT(workqueue_execute_end, #endif /* LTTNG_TRACE_WORKQUEUE_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/events/lttng-module/writeback.h b/include/instrumentation/events/lttng-module/writeback.h similarity index 99% rename from instrumentation/events/lttng-module/writeback.h rename to include/instrumentation/events/lttng-module/writeback.h index 0bdd3c82..6430dc87 100644 --- a/instrumentation/events/lttng-module/writeback.h +++ b/include/instrumentation/events/lttng-module/writeback.h @@ -5,7 +5,7 @@ #if !defined(LTTNG_TRACE_WRITEBACK_H) || defined(TRACE_HEADER_MULTI_READ) #define LTTNG_TRACE_WRITEBACK_H -#include +#include #include #include #include @@ -356,4 +356,4 @@ LTTNG_TRACEPOINT_EVENT_INSTANCE(writeback_single_inode_template, writeback_singl #endif /* LTTNG_TRACE_WRITEBACK_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/syscalls/3.0.34/powerpc-32-syscalls b/include/instrumentation/syscalls/3.0.34/powerpc-32-syscalls similarity index 100% rename from instrumentation/syscalls/3.0.34/powerpc-32-syscalls rename to include/instrumentation/syscalls/3.0.34/powerpc-32-syscalls diff --git a/instrumentation/syscalls/3.1.0-rc6/x86-32-syscalls b/include/instrumentation/syscalls/3.1.0-rc6/x86-32-syscalls similarity index 100% rename from instrumentation/syscalls/3.1.0-rc6/x86-32-syscalls rename to include/instrumentation/syscalls/3.1.0-rc6/x86-32-syscalls diff --git a/instrumentation/syscalls/3.10.0-rc7/x86-64-syscalls b/include/instrumentation/syscalls/3.10.0-rc7/x86-64-syscalls similarity index 100% rename from instrumentation/syscalls/3.10.0-rc7/x86-64-syscalls rename to include/instrumentation/syscalls/3.10.0-rc7/x86-64-syscalls diff --git a/instrumentation/syscalls/3.13.0/mips-32-syscalls b/include/instrumentation/syscalls/3.13.0/mips-32-syscalls similarity index 100% rename from instrumentation/syscalls/3.13.0/mips-32-syscalls rename to include/instrumentation/syscalls/3.13.0/mips-32-syscalls diff --git a/instrumentation/syscalls/3.18.0/mips-32-syscalls b/include/instrumentation/syscalls/3.18.0/mips-32-syscalls similarity index 100% rename from instrumentation/syscalls/3.18.0/mips-32-syscalls rename to include/instrumentation/syscalls/3.18.0/mips-32-syscalls diff --git a/instrumentation/syscalls/3.4.25/arm-32-syscalls b/include/instrumentation/syscalls/3.4.25/arm-32-syscalls similarity index 100% rename from instrumentation/syscalls/3.4.25/arm-32-syscalls rename to include/instrumentation/syscalls/3.4.25/arm-32-syscalls diff --git a/instrumentation/syscalls/3.5.0/mips-32-syscalls b/include/instrumentation/syscalls/3.5.0/mips-32-syscalls similarity index 100% rename from instrumentation/syscalls/3.5.0/mips-32-syscalls rename to include/instrumentation/syscalls/3.5.0/mips-32-syscalls diff --git a/instrumentation/syscalls/3.5.0/mips-64-syscalls b/include/instrumentation/syscalls/3.5.0/mips-64-syscalls similarity index 100% rename from instrumentation/syscalls/3.5.0/mips-64-syscalls rename to include/instrumentation/syscalls/3.5.0/mips-64-syscalls diff --git a/instrumentation/syscalls/4.4.0/arm-64-syscalls b/include/instrumentation/syscalls/4.4.0/arm-64-syscalls similarity index 100% rename from instrumentation/syscalls/4.4.0/arm-64-syscalls rename to include/instrumentation/syscalls/4.4.0/arm-64-syscalls diff --git a/instrumentation/syscalls/README b/include/instrumentation/syscalls/README similarity index 100% rename from instrumentation/syscalls/README rename to include/instrumentation/syscalls/README diff --git a/instrumentation/syscalls/headers/arm-32-syscalls_integers.h b/include/instrumentation/syscalls/headers/arm-32-syscalls_integers.h similarity index 99% rename from instrumentation/syscalls/headers/arm-32-syscalls_integers.h rename to include/instrumentation/syscalls/headers/arm-32-syscalls_integers.h index 5d0e8709..db02762e 100644 --- a/instrumentation/syscalls/headers/arm-32-syscalls_integers.h +++ b/include/instrumentation/syscalls/headers/arm-32-syscalls_integers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_INTEGERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_INTEGERS_H -#include +#include #include #include "arm-32-syscalls_integers_override.h" #include "syscalls_integers_override.h" @@ -826,7 +826,7 @@ SC_LTTNG_TRACEPOINT_EVENT(keyctl, #endif /* _TRACE_SYSCALLS_INTEGERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/arm-32-syscalls_integers_override.h b/include/instrumentation/syscalls/headers/arm-32-syscalls_integers_override.h similarity index 100% rename from instrumentation/syscalls/headers/arm-32-syscalls_integers_override.h rename to include/instrumentation/syscalls/headers/arm-32-syscalls_integers_override.h diff --git a/instrumentation/syscalls/headers/arm-32-syscalls_pointers.h b/include/instrumentation/syscalls/headers/arm-32-syscalls_pointers.h similarity index 99% rename from instrumentation/syscalls/headers/arm-32-syscalls_pointers.h rename to include/instrumentation/syscalls/headers/arm-32-syscalls_pointers.h index 094fa2ec..73122679 100644 --- a/instrumentation/syscalls/headers/arm-32-syscalls_pointers.h +++ b/include/instrumentation/syscalls/headers/arm-32-syscalls_pointers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_POINTERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_POINTERS_H -#include +#include #include #include "arm-32-syscalls_pointers_override.h" #include "syscalls_pointers_override.h" @@ -1355,7 +1355,7 @@ SC_LTTNG_TRACEPOINT_EVENT(process_vm_writev, #endif /* _TRACE_SYSCALLS_POINTERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/arm-32-syscalls_pointers_override.h b/include/instrumentation/syscalls/headers/arm-32-syscalls_pointers_override.h similarity index 100% rename from instrumentation/syscalls/headers/arm-32-syscalls_pointers_override.h rename to include/instrumentation/syscalls/headers/arm-32-syscalls_pointers_override.h diff --git a/instrumentation/syscalls/headers/arm-64-syscalls_integers.h b/include/instrumentation/syscalls/headers/arm-64-syscalls_integers.h similarity index 99% rename from instrumentation/syscalls/headers/arm-64-syscalls_integers.h rename to include/instrumentation/syscalls/headers/arm-64-syscalls_integers.h index 57d2c66b..71cc60be 100644 --- a/instrumentation/syscalls/headers/arm-64-syscalls_integers.h +++ b/include/instrumentation/syscalls/headers/arm-64-syscalls_integers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_INTEGERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_INTEGERS_H -#include +#include #include #include "arm-64-syscalls_integers_override.h" #include "syscalls_integers_override.h" @@ -693,7 +693,7 @@ SC_LTTNG_TRACEPOINT_EVENT(remap_file_pages, #endif /* _TRACE_SYSCALLS_INTEGERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/arm-64-syscalls_integers_override.h b/include/instrumentation/syscalls/headers/arm-64-syscalls_integers_override.h similarity index 100% rename from instrumentation/syscalls/headers/arm-64-syscalls_integers_override.h rename to include/instrumentation/syscalls/headers/arm-64-syscalls_integers_override.h diff --git a/instrumentation/syscalls/headers/arm-64-syscalls_pointers.h b/include/instrumentation/syscalls/headers/arm-64-syscalls_pointers.h similarity index 99% rename from instrumentation/syscalls/headers/arm-64-syscalls_pointers.h rename to include/instrumentation/syscalls/headers/arm-64-syscalls_pointers.h index f7ae855f..3b918aa8 100644 --- a/instrumentation/syscalls/headers/arm-64-syscalls_pointers.h +++ b/include/instrumentation/syscalls/headers/arm-64-syscalls_pointers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_POINTERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_POINTERS_H -#include +#include #include #include "arm-64-syscalls_pointers_override.h" #include "syscalls_pointers_override.h" @@ -1166,7 +1166,7 @@ SC_LTTNG_TRACEPOINT_EVENT(process_vm_writev, #endif /* _TRACE_SYSCALLS_POINTERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/arm-64-syscalls_pointers_override.h b/include/instrumentation/syscalls/headers/arm-64-syscalls_pointers_override.h similarity index 100% rename from instrumentation/syscalls/headers/arm-64-syscalls_pointers_override.h rename to include/instrumentation/syscalls/headers/arm-64-syscalls_pointers_override.h diff --git a/instrumentation/syscalls/headers/compat_syscalls_integers.h b/include/instrumentation/syscalls/headers/compat_syscalls_integers.h similarity index 100% rename from instrumentation/syscalls/headers/compat_syscalls_integers.h rename to include/instrumentation/syscalls/headers/compat_syscalls_integers.h diff --git a/instrumentation/syscalls/headers/compat_syscalls_pointers.h b/include/instrumentation/syscalls/headers/compat_syscalls_pointers.h similarity index 100% rename from instrumentation/syscalls/headers/compat_syscalls_pointers.h rename to include/instrumentation/syscalls/headers/compat_syscalls_pointers.h diff --git a/instrumentation/syscalls/headers/mips-32-syscalls_integers.h b/include/instrumentation/syscalls/headers/mips-32-syscalls_integers.h similarity index 99% rename from instrumentation/syscalls/headers/mips-32-syscalls_integers.h rename to include/instrumentation/syscalls/headers/mips-32-syscalls_integers.h index 69338a12..cfaa1d88 100644 --- a/instrumentation/syscalls/headers/mips-32-syscalls_integers.h +++ b/include/instrumentation/syscalls/headers/mips-32-syscalls_integers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_INTEGERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_INTEGERS_H -#include +#include #include #include "mips-32-syscalls_integers_override.h" #include "syscalls_integers_override.h" @@ -758,7 +758,7 @@ SC_LTTNG_TRACEPOINT_EVENT(mips_mmap2, #endif /* _TRACE_SYSCALLS_INTEGERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/mips-32-syscalls_integers_override.h b/include/instrumentation/syscalls/headers/mips-32-syscalls_integers_override.h similarity index 100% rename from instrumentation/syscalls/headers/mips-32-syscalls_integers_override.h rename to include/instrumentation/syscalls/headers/mips-32-syscalls_integers_override.h diff --git a/instrumentation/syscalls/headers/mips-32-syscalls_pointers.h b/include/instrumentation/syscalls/headers/mips-32-syscalls_pointers.h similarity index 99% rename from instrumentation/syscalls/headers/mips-32-syscalls_pointers.h rename to include/instrumentation/syscalls/headers/mips-32-syscalls_pointers.h index d638087b..e2cd9604 100644 --- a/instrumentation/syscalls/headers/mips-32-syscalls_pointers.h +++ b/include/instrumentation/syscalls/headers/mips-32-syscalls_pointers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_POINTERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_POINTERS_H -#include +#include #include #include "mips-32-syscalls_pointers_override.h" #include "syscalls_pointers_override.h" @@ -1327,7 +1327,7 @@ SC_LTTNG_TRACEPOINT_EVENT(epoll_pwait, #endif /* _TRACE_SYSCALLS_POINTERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/mips-32-syscalls_pointers_override.h b/include/instrumentation/syscalls/headers/mips-32-syscalls_pointers_override.h similarity index 100% rename from instrumentation/syscalls/headers/mips-32-syscalls_pointers_override.h rename to include/instrumentation/syscalls/headers/mips-32-syscalls_pointers_override.h diff --git a/instrumentation/syscalls/headers/mips-64-syscalls_integers.h b/include/instrumentation/syscalls/headers/mips-64-syscalls_integers.h similarity index 99% rename from instrumentation/syscalls/headers/mips-64-syscalls_integers.h rename to include/instrumentation/syscalls/headers/mips-64-syscalls_integers.h index 302a788d..814ee3a6 100644 --- a/instrumentation/syscalls/headers/mips-64-syscalls_integers.h +++ b/include/instrumentation/syscalls/headers/mips-64-syscalls_integers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_INTEGERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_INTEGERS_H -#include +#include #include #include "mips-64-syscalls_integers_override.h" #include "syscalls_integers_override.h" @@ -796,7 +796,7 @@ SC_LTTNG_TRACEPOINT_EVENT(mips_mmap, #endif /* _TRACE_SYSCALLS_INTEGERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/mips-64-syscalls_integers_override.h b/include/instrumentation/syscalls/headers/mips-64-syscalls_integers_override.h similarity index 100% rename from instrumentation/syscalls/headers/mips-64-syscalls_integers_override.h rename to include/instrumentation/syscalls/headers/mips-64-syscalls_integers_override.h diff --git a/instrumentation/syscalls/headers/mips-64-syscalls_pointers.h b/include/instrumentation/syscalls/headers/mips-64-syscalls_pointers.h similarity index 99% rename from instrumentation/syscalls/headers/mips-64-syscalls_pointers.h rename to include/instrumentation/syscalls/headers/mips-64-syscalls_pointers.h index 8bcf9b00..290f9c3e 100644 --- a/instrumentation/syscalls/headers/mips-64-syscalls_pointers.h +++ b/include/instrumentation/syscalls/headers/mips-64-syscalls_pointers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_POINTERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_POINTERS_H -#include +#include #include #include "mips-64-syscalls_pointers_override.h" #include "syscalls_pointers_override.h" @@ -1306,7 +1306,7 @@ SC_LTTNG_TRACEPOINT_EVENT(splice, #endif /* _TRACE_SYSCALLS_POINTERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/mips-64-syscalls_pointers_override.h b/include/instrumentation/syscalls/headers/mips-64-syscalls_pointers_override.h similarity index 100% rename from instrumentation/syscalls/headers/mips-64-syscalls_pointers_override.h rename to include/instrumentation/syscalls/headers/mips-64-syscalls_pointers_override.h diff --git a/instrumentation/syscalls/headers/powerpc-32-syscalls_integers.h b/include/instrumentation/syscalls/headers/powerpc-32-syscalls_integers.h similarity index 99% rename from instrumentation/syscalls/headers/powerpc-32-syscalls_integers.h rename to include/instrumentation/syscalls/headers/powerpc-32-syscalls_integers.h index a4ee13e9..d84916af 100644 --- a/instrumentation/syscalls/headers/powerpc-32-syscalls_integers.h +++ b/include/instrumentation/syscalls/headers/powerpc-32-syscalls_integers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_INTEGERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_INTEGERS_H -#include +#include #include #include "powerpc-32-syscalls_integers_override.h" #include "syscalls_integers_override.h" @@ -726,7 +726,7 @@ SC_LTTNG_TRACEPOINT_EVENT(keyctl, #endif /* _TRACE_SYSCALLS_INTEGERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/powerpc-32-syscalls_integers_override.h b/include/instrumentation/syscalls/headers/powerpc-32-syscalls_integers_override.h similarity index 100% rename from instrumentation/syscalls/headers/powerpc-32-syscalls_integers_override.h rename to include/instrumentation/syscalls/headers/powerpc-32-syscalls_integers_override.h diff --git a/instrumentation/syscalls/headers/powerpc-32-syscalls_pointers.h b/include/instrumentation/syscalls/headers/powerpc-32-syscalls_pointers.h similarity index 99% rename from instrumentation/syscalls/headers/powerpc-32-syscalls_pointers.h rename to include/instrumentation/syscalls/headers/powerpc-32-syscalls_pointers.h index 73a28a79..2fcf12cd 100644 --- a/instrumentation/syscalls/headers/powerpc-32-syscalls_pointers.h +++ b/include/instrumentation/syscalls/headers/powerpc-32-syscalls_pointers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_POINTERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_POINTERS_H -#include +#include #include #include "powerpc-32-syscalls_pointers_override.h" #include "syscalls_pointers_override.h" @@ -1355,7 +1355,7 @@ SC_LTTNG_TRACEPOINT_EVENT(recvfrom, #endif /* _TRACE_SYSCALLS_POINTERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/powerpc-32-syscalls_pointers_override.h b/include/instrumentation/syscalls/headers/powerpc-32-syscalls_pointers_override.h similarity index 100% rename from instrumentation/syscalls/headers/powerpc-32-syscalls_pointers_override.h rename to include/instrumentation/syscalls/headers/powerpc-32-syscalls_pointers_override.h diff --git a/instrumentation/syscalls/headers/syscalls_integers.h b/include/instrumentation/syscalls/headers/syscalls_integers.h similarity index 100% rename from instrumentation/syscalls/headers/syscalls_integers.h rename to include/instrumentation/syscalls/headers/syscalls_integers.h diff --git a/instrumentation/syscalls/headers/syscalls_integers_override.h b/include/instrumentation/syscalls/headers/syscalls_integers_override.h similarity index 100% rename from instrumentation/syscalls/headers/syscalls_integers_override.h rename to include/instrumentation/syscalls/headers/syscalls_integers_override.h diff --git a/instrumentation/syscalls/headers/syscalls_pointers.h b/include/instrumentation/syscalls/headers/syscalls_pointers.h similarity index 100% rename from instrumentation/syscalls/headers/syscalls_pointers.h rename to include/instrumentation/syscalls/headers/syscalls_pointers.h diff --git a/instrumentation/syscalls/headers/syscalls_pointers_override.h b/include/instrumentation/syscalls/headers/syscalls_pointers_override.h similarity index 100% rename from instrumentation/syscalls/headers/syscalls_pointers_override.h rename to include/instrumentation/syscalls/headers/syscalls_pointers_override.h diff --git a/instrumentation/syscalls/headers/syscalls_unknown.h b/include/instrumentation/syscalls/headers/syscalls_unknown.h similarity index 94% rename from instrumentation/syscalls/headers/syscalls_unknown.h rename to include/instrumentation/syscalls/headers/syscalls_unknown.h index 1718f5c0..4e9d8608 100644 --- a/instrumentation/syscalls/headers/syscalls_unknown.h +++ b/include/instrumentation/syscalls/headers/syscalls_unknown.h @@ -3,7 +3,7 @@ #if !defined(_TRACE_SYSCALLS_UNKNOWN_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_UNKNOWN_H -#include +#include #include #define UNKNOWN_SYSCALL_NRARGS 6 @@ -52,4 +52,4 @@ LTTNG_TRACEPOINT_EVENT(compat_syscall_exit_unknown, #endif /* _TRACE_SYSCALLS_UNKNOWN_H */ /* This part must be outside protection */ -#include +#include diff --git a/instrumentation/syscalls/headers/x86-32-syscalls_integers.h b/include/instrumentation/syscalls/headers/x86-32-syscalls_integers.h similarity index 99% rename from instrumentation/syscalls/headers/x86-32-syscalls_integers.h rename to include/instrumentation/syscalls/headers/x86-32-syscalls_integers.h index 6e1fc8ec..4b1a5040 100644 --- a/instrumentation/syscalls/headers/x86-32-syscalls_integers.h +++ b/include/instrumentation/syscalls/headers/x86-32-syscalls_integers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_INTEGERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_INTEGERS_H -#include +#include #include #include "x86-32-syscalls_integers_override.h" #include "syscalls_integers_override.h" @@ -822,7 +822,7 @@ SC_LTTNG_TRACEPOINT_EVENT(mmap_pgoff, #endif /* _TRACE_SYSCALLS_INTEGERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/x86-32-syscalls_integers_override.h b/include/instrumentation/syscalls/headers/x86-32-syscalls_integers_override.h similarity index 100% rename from instrumentation/syscalls/headers/x86-32-syscalls_integers_override.h rename to include/instrumentation/syscalls/headers/x86-32-syscalls_integers_override.h diff --git a/instrumentation/syscalls/headers/x86-32-syscalls_pointers.h b/include/instrumentation/syscalls/headers/x86-32-syscalls_pointers.h similarity index 99% rename from instrumentation/syscalls/headers/x86-32-syscalls_pointers.h rename to include/instrumentation/syscalls/headers/x86-32-syscalls_pointers.h index da15bfd1..34ea4d0d 100644 --- a/instrumentation/syscalls/headers/x86-32-syscalls_pointers.h +++ b/include/instrumentation/syscalls/headers/x86-32-syscalls_pointers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_POINTERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_POINTERS_H -#include +#include #include #include "x86-32-syscalls_pointers_override.h" #include "syscalls_pointers_override.h" @@ -1306,7 +1306,7 @@ SC_LTTNG_TRACEPOINT_EVENT(epoll_pwait, #endif /* _TRACE_SYSCALLS_POINTERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/x86-32-syscalls_pointers_override.h b/include/instrumentation/syscalls/headers/x86-32-syscalls_pointers_override.h similarity index 100% rename from instrumentation/syscalls/headers/x86-32-syscalls_pointers_override.h rename to include/instrumentation/syscalls/headers/x86-32-syscalls_pointers_override.h diff --git a/instrumentation/syscalls/headers/x86-64-syscalls_integers.h b/include/instrumentation/syscalls/headers/x86-64-syscalls_integers.h similarity index 99% rename from instrumentation/syscalls/headers/x86-64-syscalls_integers.h rename to include/instrumentation/syscalls/headers/x86-64-syscalls_integers.h index c2963921..3fa8fbff 100644 --- a/instrumentation/syscalls/headers/x86-64-syscalls_integers.h +++ b/include/instrumentation/syscalls/headers/x86-64-syscalls_integers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_INTEGERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_INTEGERS_H -#include +#include #include #include "x86-64-syscalls_integers_override.h" #include "syscalls_integers_override.h" @@ -751,7 +751,7 @@ SC_LTTNG_TRACEPOINT_EVENT(mmap, #endif /* _TRACE_SYSCALLS_INTEGERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/x86-64-syscalls_integers_override.h b/include/instrumentation/syscalls/headers/x86-64-syscalls_integers_override.h similarity index 100% rename from instrumentation/syscalls/headers/x86-64-syscalls_integers_override.h rename to include/instrumentation/syscalls/headers/x86-64-syscalls_integers_override.h diff --git a/instrumentation/syscalls/headers/x86-64-syscalls_pointers.h b/include/instrumentation/syscalls/headers/x86-64-syscalls_pointers.h similarity index 99% rename from instrumentation/syscalls/headers/x86-64-syscalls_pointers.h rename to include/instrumentation/syscalls/headers/x86-64-syscalls_pointers.h index 0d084882..e57a83a0 100644 --- a/instrumentation/syscalls/headers/x86-64-syscalls_pointers.h +++ b/include/instrumentation/syscalls/headers/x86-64-syscalls_pointers.h @@ -9,7 +9,7 @@ #if !defined(_TRACE_SYSCALLS_POINTERS_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_POINTERS_H -#include +#include #include #include "x86-64-syscalls_pointers_override.h" #include "syscalls_pointers_override.h" @@ -1348,7 +1348,7 @@ SC_LTTNG_TRACEPOINT_EVENT(process_vm_writev, #endif /* _TRACE_SYSCALLS_POINTERS_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/headers/x86-64-syscalls_pointers_override.h b/include/instrumentation/syscalls/headers/x86-64-syscalls_pointers_override.h similarity index 100% rename from instrumentation/syscalls/headers/x86-64-syscalls_pointers_override.h rename to include/instrumentation/syscalls/headers/x86-64-syscalls_pointers_override.h diff --git a/instrumentation/syscalls/lttng-get-syscall-inout.sh b/include/instrumentation/syscalls/lttng-get-syscall-inout.sh similarity index 100% rename from instrumentation/syscalls/lttng-get-syscall-inout.sh rename to include/instrumentation/syscalls/lttng-get-syscall-inout.sh diff --git a/instrumentation/syscalls/lttng-syscalls-extract.sh b/include/instrumentation/syscalls/lttng-syscalls-extract.sh similarity index 100% rename from instrumentation/syscalls/lttng-syscalls-extract.sh rename to include/instrumentation/syscalls/lttng-syscalls-extract.sh diff --git a/instrumentation/syscalls/lttng-syscalls-extractor/Makefile b/include/instrumentation/syscalls/lttng-syscalls-extractor/Makefile similarity index 100% rename from instrumentation/syscalls/lttng-syscalls-extractor/Makefile rename to include/instrumentation/syscalls/lttng-syscalls-extractor/Makefile diff --git a/instrumentation/syscalls/lttng-syscalls-extractor/linux-link-trace-syscalls-as-data.patch b/include/instrumentation/syscalls/lttng-syscalls-extractor/linux-link-trace-syscalls-as-data.patch similarity index 100% rename from instrumentation/syscalls/lttng-syscalls-extractor/linux-link-trace-syscalls-as-data.patch rename to include/instrumentation/syscalls/lttng-syscalls-extractor/linux-link-trace-syscalls-as-data.patch diff --git a/instrumentation/syscalls/lttng-syscalls-extractor/lttng-syscalls-extractor.c b/include/instrumentation/syscalls/lttng-syscalls-extractor/lttng-syscalls-extractor.c similarity index 100% rename from instrumentation/syscalls/lttng-syscalls-extractor/lttng-syscalls-extractor.c rename to include/instrumentation/syscalls/lttng-syscalls-extractor/lttng-syscalls-extractor.c diff --git a/instrumentation/syscalls/lttng-syscalls-generate-headers.sh b/include/instrumentation/syscalls/lttng-syscalls-generate-headers.sh similarity index 99% rename from instrumentation/syscalls/lttng-syscalls-generate-headers.sh rename to include/instrumentation/syscalls/lttng-syscalls-generate-headers.sh index 863f390e..4216dbd1 100755 --- a/instrumentation/syscalls/lttng-syscalls-generate-headers.sh +++ b/include/instrumentation/syscalls/lttng-syscalls-generate-headers.sh @@ -81,7 +81,7 @@ echo "/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ #if !defined(_TRACE_SYSCALLS_${CLASSCAP}_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_SYSCALLS_${CLASSCAP}_H -#include +#include #include #include \"${INPUTFILE}_${CLASS}_override.h\" #include \"syscalls_${CLASS}_override.h\" @@ -290,7 +290,7 @@ echo \ #endif /* _TRACE_SYSCALLS_${CLASSCAP}_H */ /* This part must be outside protection */ -#include +#include #else /* CREATE_SYSCALL_TABLE */ diff --git a/instrumentation/syscalls/table-syscall-inout.txt b/include/instrumentation/syscalls/table-syscall-inout.txt similarity index 100% rename from instrumentation/syscalls/table-syscall-inout.txt rename to include/instrumentation/syscalls/table-syscall-inout.txt diff --git a/lib/align.h b/include/lttng/align.h similarity index 97% rename from lib/align.h rename to include/lttng/align.h index c0695494..964ed1a8 100644 --- a/lib/align.h +++ b/include/lttng/align.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) * - * lib/align.h + * lttng/align.h * * Copyright (C) 2010-2012 Mathieu Desnoyers */ @@ -11,7 +11,7 @@ #ifdef __KERNEL__ #include -#include +#include #define ALIGN_FLOOR(x, a) __ALIGN_FLOOR_MASK(x, (typeof(x)) (a) - 1) #define __ALIGN_FLOOR_MASK(x, mask) ((x) & ~(mask)) diff --git a/lib/bitfield.h b/include/lttng/bitfield.h similarity index 99% rename from lib/bitfield.h rename to include/lttng/bitfield.h index 7d5eaeaa..b0fcbd25 100644 --- a/lib/bitfield.h +++ b/include/lttng/bitfield.h @@ -7,7 +7,7 @@ #define _BABELTRACE_BITFIELD_H #include -#include +#include #ifndef CHAR_BIT #define CHAR_BIT 8 diff --git a/lib/bug.h b/include/lttng/bug.h similarity index 98% rename from lib/bug.h rename to include/lttng/bug.h index ea73f158..0a2c5e4b 100644 --- a/lib/bug.h +++ b/include/lttng/bug.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) * - * lib/bug.h + * lttng/bug.h * * Copyright (C) 2010-2012 Mathieu Desnoyers */ diff --git a/probes/define_trace.h b/include/lttng/define_trace.h similarity index 98% rename from probes/define_trace.h rename to include/lttng/define_trace.h index f8f9d342..a97a66ab 100644 --- a/probes/define_trace.h +++ b/include/lttng/define_trace.h @@ -84,7 +84,7 @@ #define DECLARE_TRACE(name, proto, args) #ifdef LTTNG_PACKAGE_BUILD -#include +#include #endif #undef LTTNG_TRACEPOINT_EVENT diff --git a/filter-bytecode.h b/include/lttng/filter-bytecode.h similarity index 100% rename from filter-bytecode.h rename to include/lttng/filter-bytecode.h diff --git a/lttng-abi-old.h b/include/lttng/lttng-abi-old.h similarity index 99% rename from lttng-abi-old.h rename to include/lttng/lttng-abi-old.h index 0b1c7b11..2444ca5b 100644 --- a/lttng-abi-old.h +++ b/include/lttng/lttng-abi-old.h @@ -11,7 +11,7 @@ #define _LTTNG_ABI_OLD_H #include -#include +#include /* * LTTng DebugFS ABI structures. diff --git a/lttng-abi.h b/include/lttng/lttng-abi.h similarity index 100% rename from lttng-abi.h rename to include/lttng/lttng-abi.h diff --git a/lttng-clock.h b/include/lttng/lttng-clock.h similarity index 100% rename from lttng-clock.h rename to include/lttng/lttng-clock.h diff --git a/lttng-cpuhotplug.h b/include/lttng/lttng-cpuhotplug.h similarity index 100% rename from lttng-cpuhotplug.h rename to include/lttng/lttng-cpuhotplug.h diff --git a/lttng-endian.h b/include/lttng/lttng-endian.h similarity index 100% rename from lttng-endian.h rename to include/lttng/lttng-endian.h diff --git a/probes/lttng-events-nowrite.h b/include/lttng/lttng-events-nowrite.h similarity index 100% rename from probes/lttng-events-nowrite.h rename to include/lttng/lttng-events-nowrite.h diff --git a/probes/lttng-events-reset.h b/include/lttng/lttng-events-reset.h similarity index 100% rename from probes/lttng-events-reset.h rename to include/lttng/lttng-events-reset.h diff --git a/probes/lttng-events-write.h b/include/lttng/lttng-events-write.h similarity index 100% rename from probes/lttng-events-write.h rename to include/lttng/lttng-events-write.h diff --git a/lttng-events.h b/include/lttng/lttng-events.h similarity index 99% rename from lttng-events.h rename to include/lttng/lttng-events.h index 69775307..026e4461 100644 --- a/lttng-events.h +++ b/include/lttng/lttng-events.h @@ -14,13 +14,13 @@ #include #include #include -#include +#include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #define lttng_is_signed_type(type) (((type)(-1)) < 0) diff --git a/lttng-filter.h b/include/lttng/lttng-filter.h similarity index 98% rename from lttng-filter.h rename to include/lttng/lttng-filter.h index fec2db1f..d3b9aacd 100644 --- a/lttng-filter.h +++ b/include/lttng/lttng-filter.h @@ -12,8 +12,8 @@ #include -#include -#include +#include +#include /* Filter stack length, in number of entries */ #define FILTER_STACK_LEN 10 /* includes 2 dummy */ diff --git a/lttng-kernel-version.h b/include/lttng/lttng-kernel-version.h similarity index 100% rename from lttng-kernel-version.h rename to include/lttng/lttng-kernel-version.h diff --git a/probes/lttng-probe-user.h b/include/lttng/lttng-probe-user.h similarity index 100% rename from probes/lttng-probe-user.h rename to include/lttng/lttng-probe-user.h diff --git a/lttng-string-utils.h b/include/lttng/lttng-string-utils.h similarity index 100% rename from lttng-string-utils.h rename to include/lttng/lttng-string-utils.h diff --git a/lttng-tp-mempool.h b/include/lttng/lttng-tp-mempool.h similarity index 100% rename from lttng-tp-mempool.h rename to include/lttng/lttng-tp-mempool.h diff --git a/probes/lttng-tracepoint-event-impl.h b/include/lttng/lttng-tracepoint-event-impl.h similarity index 97% rename from probes/lttng-tracepoint-event-impl.h rename to include/lttng/lttng-tracepoint-event-impl.h index 8584fbe0..5779bac0 100644 --- a/probes/lttng-tracepoint-event-impl.h +++ b/include/lttng/lttng-tracepoint-event-impl.h @@ -12,18 +12,20 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include #include -#include -#include -#include +#include +#include +#include #define __LTTNG_NULL_STRING "(null)" +#undef PARAMS +#define PARAMS(args...) args + /* * Macro declarations used for all stages. */ @@ -116,7 +118,7 @@ */ /* Reset all macros within TRACEPOINT_EVENT */ -#include +#include #undef TP_PROTO #define TP_PROTO(...) __VA_ARGS__ @@ -142,7 +144,7 @@ void trace_##_name(void); * class and the instance using the class actually match. */ -#include /* Reset all macros within TRACE_EVENT */ +#include /* Reset all macros within TRACE_EVENT */ #undef TP_PROTO #define TP_PROTO(...) __VA_ARGS__ @@ -173,7 +175,7 @@ void __event_template_proto___##_name(void); * * Unfolding the enums */ -#include /* Reset all macros within TRACE_EVENT */ +#include /* Reset all macros within TRACE_EVENT */ /* Enumeration entry (single value) */ #undef ctf_enum_value @@ -247,9 +249,9 @@ void __event_template_proto___##_name(void); */ /* Reset all macros within TRACEPOINT_EVENT */ -#include -#include -#include +#include +#include +#include #undef _ctf_integer_ext #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \ @@ -450,7 +452,7 @@ void __event_template_proto___##_name(void); */ /* Reset all macros within TRACEPOINT_EVENT */ -#include +#include #undef TP_PROTO #define TP_PROTO(...) __VA_ARGS__ @@ -472,8 +474,8 @@ static void __event_probe__##_name(void *__data); */ /* Reset all macros within TRACEPOINT_EVENT */ -#include -#include +#include +#include #undef _ctf_integer_ext #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \ @@ -598,9 +600,9 @@ error: \ */ /* Reset all macros within TRACEPOINT_EVENT */ -#include -#include -#include +#include +#include +#include #undef _ctf_integer_ext_fetched #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \ @@ -789,8 +791,8 @@ void __event_prepare_filter_stack__##_name(char *__stack_data, \ */ /* Reset all macros within TRACEPOINT_EVENT */ -#include -#include +#include +#include #undef _ctf_integer_ext #define _ctf_integer_ext(_type, _item, _src, _byte_order, _base, _user, _nowrite) \ @@ -878,8 +880,8 @@ static inline size_t __event_get_align__##_name(void *__tp_locvar) \ */ /* Reset all macros within TRACEPOINT_EVENT */ -#include -#include +#include +#include #undef _ctf_integer_ext_fetched #define _ctf_integer_ext_fetched(_type, _item, _src, _byte_order, _base, _nowrite) \ @@ -1323,7 +1325,7 @@ __post: \ /* Named field types must be defined in lttng-types.h */ -#include /* Reset all macros within LTTNG_TRACEPOINT_EVENT */ +#include /* Reset all macros within LTTNG_TRACEPOINT_EVENT */ #ifndef TP_PROBE_CB #define TP_PROBE_CB(_template) &__event_probe__##_template @@ -1352,7 +1354,7 @@ static const struct lttng_event_desc __event_desc___##_map = { \ * Create an array of event description pointers. */ -#include /* Reset all macros within LTTNG_TRACEPOINT_EVENT */ +#include /* Reset all macros within LTTNG_TRACEPOINT_EVENT */ #undef LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS #define LTTNG_TRACEPOINT_EVENT_INSTANCE_MAP_NOARGS(_template, _name, _map) \ @@ -1400,7 +1402,7 @@ static __used struct lttng_probe_desc TP_ID(__probe_desc___, TRACE_SYSTEM) = { * Register/unregister probes at module load/unload. */ -#include /* Reset all macros within LTTNG_TRACEPOINT_EVENT */ +#include /* Reset all macros within LTTNG_TRACEPOINT_EVENT */ #define TP_ID1(_token, _system) _token##_system #define TP_ID(_token, _system) TP_ID1(_token, _system) diff --git a/probes/lttng-tracepoint-event.h b/include/lttng/lttng-tracepoint-event.h similarity index 100% rename from probes/lttng-tracepoint-event.h rename to include/lttng/lttng-tracepoint-event.h diff --git a/lttng-tracepoint.h b/include/lttng/lttng-tracepoint.h similarity index 100% rename from lttng-tracepoint.h rename to include/lttng/lttng-tracepoint.h diff --git a/lttng-tracer-core.h b/include/lttng/lttng-tracer-core.h similarity index 93% rename from lttng-tracer-core.h rename to include/lttng/lttng-tracer-core.h index ad82ae2c..18716738 100644 --- a/lttng-tracer-core.h +++ b/include/lttng/lttng-tracer-core.h @@ -19,7 +19,7 @@ #define RING_BUFFER_ALIGN #endif -#include +#include struct lttng_session; struct lttng_channel; diff --git a/lttng-tracer.h b/include/lttng/lttng-tracer.h similarity index 96% rename from lttng-tracer.h rename to include/lttng/lttng-tracer.h index 1712d031..322b93e2 100644 --- a/lttng-tracer.h +++ b/include/lttng/lttng-tracer.h @@ -22,8 +22,8 @@ #include #include -#include -#include +#include +#include #define LTTNG_MODULES_MAJOR_VERSION 2 #define LTTNG_MODULES_MINOR_VERSION 12 diff --git a/probes/lttng-types.h b/include/lttng/lttng-types.h similarity index 90% rename from probes/lttng-types.h rename to include/lttng/lttng-types.h index f6b4785f..2de88735 100644 --- a/probes/lttng-types.h +++ b/include/lttng/lttng-types.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) * - * probes/lttng-types.h + * lttng/lttng-types.h * * LTTng types. * @@ -15,13 +15,15 @@ #define _LTTNG_PROBES_LTTNG_TYPES_H #include -#include -#include -#include -#include +#include +#include +#include #endif /* _LTTNG_PROBES_LTTNG_TYPES_H */ +#undef PARAMS +#define PARAMS(args...) args + /* Export enumerations */ #ifdef STAGE_EXPORT_ENUMS diff --git a/lib/prio_heap/lttng_prio_heap.h b/include/lttng/lttng_prio_heap.h similarity index 100% rename from lib/prio_heap/lttng_prio_heap.h rename to include/lttng/lttng_prio_heap.h diff --git a/include/ringbuffer/api.h b/include/ringbuffer/api.h index 73aa8746..3cd9f29a 100644 --- a/include/ringbuffer/api.h +++ b/include/ringbuffer/api.h @@ -1,2 +1,24 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/api.h + * + * Ring Buffer API. + * + * Copyright (C) 2010-2012 Mathieu Desnoyers + */ + +#ifndef _LIB_RING_BUFFER_API_H +#define _LIB_RING_BUFFER_API_H + +#include +#include +#include + +/* + * ring_buffer_frontend_api.h contains static inline functions that depend on + * client static inlines. Hence the inclusion of this >api> header only + * within the client. + */ +#include + +#endif /* _LIB_RING_BUFFER_API_H */ diff --git a/include/ringbuffer/backend.h b/include/ringbuffer/backend.h index 48641606..d01400da 100644 --- a/include/ringbuffer/backend.h +++ b/include/ringbuffer/backend.h @@ -1,2 +1,463 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/backend.h + * + * Ring buffer backend (API). + * + * Copyright (C) 2010-2012 Mathieu Desnoyers + * + * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by + * the reader in flight recorder mode. + */ + +#ifndef _LIB_RING_BUFFER_BACKEND_H +#define _LIB_RING_BUFFER_BACKEND_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Internal helpers */ +#include +#include + +/* Ring buffer backend API */ + +/* Ring buffer backend access (read/write) */ + +extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb, + size_t offset, void *dest, size_t len); + +extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb, + size_t offset, void __user *dest, + size_t len); + +extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb, + size_t offset, void *dest, size_t len); + +extern unsigned long * +lib_ring_buffer_read_get_pfn(struct lib_ring_buffer_backend *bufb, size_t offset, + void ***virt); + +/* + * Return the address where a given offset is located. + * Should be used to get the current subbuffer header pointer. Given we know + * it's never on a page boundary, it's safe to write directly to this address, + * as long as the write is never bigger than a page size. + */ +extern void * +lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb, + size_t offset); +extern void * +lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb, + size_t offset); + +/** + * lib_ring_buffer_write - write data to a buffer backend + * @config : ring buffer instance configuration + * @ctx: ring buffer context. (input arguments only) + * @src : source pointer to copy from + * @len : length of data to copy + * + * This function copies "len" bytes of data from a source pointer to a buffer + * backend, at the current context offset. This is more or less a buffer + * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write) + * if copy is crossing a page boundary. + */ +static inline __attribute__((always_inline)) +void lib_ring_buffer_write(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx, + const void *src, size_t len) +{ + struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; + struct channel_backend *chanb = &ctx->chan->backend; + size_t index, pagecpy; + size_t offset = ctx->buf_offset; + struct lib_ring_buffer_backend_pages *backend_pages; + + if (unlikely(!len)) + return; + backend_pages = + lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); + offset &= chanb->buf_size - 1; + index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; + pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); + if (likely(pagecpy == len)) + lib_ring_buffer_do_copy(config, + backend_pages->p[index].virt + + (offset & ~PAGE_MASK), + src, len); + else + _lib_ring_buffer_write(bufb, offset, src, len, 0); + ctx->buf_offset += len; +} + +/** + * lib_ring_buffer_memset - write len bytes of c to a buffer backend + * @config : ring buffer instance configuration + * @bufb : ring buffer backend + * @offset : offset within the buffer + * @c : the byte to copy + * @len : number of bytes to copy + * + * This function writes "len" bytes of "c" to a buffer backend, at a specific + * offset. This is more or less a buffer backend-specific memset() operation. + * Calls the slow path (_ring_buffer_memset) if write is crossing a page + * boundary. + */ +static inline +void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx, int c, size_t len) +{ + + struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; + struct channel_backend *chanb = &ctx->chan->backend; + size_t index, pagecpy; + size_t offset = ctx->buf_offset; + struct lib_ring_buffer_backend_pages *backend_pages; + + if (unlikely(!len)) + return; + backend_pages = + lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); + offset &= chanb->buf_size - 1; + index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; + pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); + if (likely(pagecpy == len)) + lib_ring_buffer_do_memset(backend_pages->p[index].virt + + (offset & ~PAGE_MASK), + c, len); + else + _lib_ring_buffer_memset(bufb, offset, c, len, 0); + ctx->buf_offset += len; +} + +/* + * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL + * terminating character is found in @src. Returns the number of bytes + * copied. Does *not* terminate @dest with NULL terminating character. + */ +static inline __attribute__((always_inline)) +size_t lib_ring_buffer_do_strcpy(const struct lib_ring_buffer_config *config, + char *dest, const char *src, size_t len) +{ + size_t count; + + for (count = 0; count < len; count++) { + char c; + + /* + * Only read source character once, in case it is + * modified concurrently. + */ + c = READ_ONCE(src[count]); + if (!c) + break; + lib_ring_buffer_do_copy(config, &dest[count], &c, 1); + } + return count; +} + +/* + * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL + * terminating character is found in @src, or when a fault occurs. + * Returns the number of bytes copied. Does *not* terminate @dest with + * NULL terminating character. + * + * This function deals with userspace pointers, it should never be called + * directly without having the src pointer checked with access_ok() + * previously. + */ +static inline __attribute__((always_inline)) +size_t lib_ring_buffer_do_strcpy_from_user_inatomic(const struct lib_ring_buffer_config *config, + char *dest, const char __user *src, size_t len) +{ + size_t count; + + for (count = 0; count < len; count++) { + int ret; + char c; + + ret = __copy_from_user_inatomic(&c, src + count, 1); + if (ret || !c) + break; + lib_ring_buffer_do_copy(config, &dest[count], &c, 1); + } + return count; +} + +/** + * lib_ring_buffer_strcpy - write string data to a buffer backend + * @config : ring buffer instance configuration + * @ctx: ring buffer context. (input arguments only) + * @src : source pointer to copy from + * @len : length of data to copy + * @pad : character to use for padding + * + * This function copies @len - 1 bytes of string data from a source + * pointer to a buffer backend, followed by a terminating '\0' + * character, at the current context offset. This is more or less a + * buffer backend-specific strncpy() operation. If a terminating '\0' + * character is found in @src before @len - 1 characters are copied, pad + * the buffer with @pad characters (e.g. '#'). Calls the slow path + * (_ring_buffer_strcpy) if copy is crossing a page boundary. + */ +static inline +void lib_ring_buffer_strcpy(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx, + const char *src, size_t len, int pad) +{ + struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; + struct channel_backend *chanb = &ctx->chan->backend; + size_t index, pagecpy; + size_t offset = ctx->buf_offset; + struct lib_ring_buffer_backend_pages *backend_pages; + + if (unlikely(!len)) + return; + backend_pages = + lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); + offset &= chanb->buf_size - 1; + index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; + pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); + if (likely(pagecpy == len)) { + size_t count; + + count = lib_ring_buffer_do_strcpy(config, + backend_pages->p[index].virt + + (offset & ~PAGE_MASK), + src, len - 1); + offset += count; + /* Padding */ + if (unlikely(count < len - 1)) { + size_t pad_len = len - 1 - count; + + lib_ring_buffer_do_memset(backend_pages->p[index].virt + + (offset & ~PAGE_MASK), + pad, pad_len); + offset += pad_len; + } + /* Ending '\0' */ + lib_ring_buffer_do_memset(backend_pages->p[index].virt + + (offset & ~PAGE_MASK), + '\0', 1); + } else { + _lib_ring_buffer_strcpy(bufb, offset, src, len, 0, pad); + } + ctx->buf_offset += len; +} + +/** + * lib_ring_buffer_copy_from_user_inatomic - write userspace data to a buffer backend + * @config : ring buffer instance configuration + * @ctx: ring buffer context. (input arguments only) + * @src : userspace source pointer to copy from + * @len : length of data to copy + * + * This function copies "len" bytes of data from a userspace pointer to a + * buffer backend, at the current context offset. This is more or less a buffer + * backend-specific memcpy() operation. Calls the slow path + * (_ring_buffer_write_from_user_inatomic) if copy is crossing a page boundary. + * Disable the page fault handler to ensure we never try to take the mmap_sem. + */ +static inline __attribute__((always_inline)) +void lib_ring_buffer_copy_from_user_inatomic(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx, + const void __user *src, size_t len) +{ + struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; + struct channel_backend *chanb = &ctx->chan->backend; + size_t index, pagecpy; + size_t offset = ctx->buf_offset; + struct lib_ring_buffer_backend_pages *backend_pages; + unsigned long ret; + mm_segment_t old_fs = get_fs(); + + if (unlikely(!len)) + return; + backend_pages = + lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); + offset &= chanb->buf_size - 1; + index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; + pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); + + set_fs(KERNEL_DS); + pagefault_disable(); + if (unlikely(!access_ok(src, len))) + goto fill_buffer; + + if (likely(pagecpy == len)) { + ret = lib_ring_buffer_do_copy_from_user_inatomic( + backend_pages->p[index].virt + (offset & ~PAGE_MASK), + src, len); + if (unlikely(ret > 0)) { + /* Copy failed. */ + goto fill_buffer; + } + } else { + _lib_ring_buffer_copy_from_user_inatomic(bufb, offset, src, len, 0); + } + pagefault_enable(); + set_fs(old_fs); + ctx->buf_offset += len; + + return; + +fill_buffer: + pagefault_enable(); + set_fs(old_fs); + /* + * In the error path we call the slow path version to avoid + * the pollution of static inline code. + */ + _lib_ring_buffer_memset(bufb, offset, 0, len, 0); +} + +/** + * lib_ring_buffer_strcpy_from_user_inatomic - write userspace string data to a buffer backend + * @config : ring buffer instance configuration + * @ctx: ring buffer context (input arguments only) + * @src : userspace source pointer to copy from + * @len : length of data to copy + * @pad : character to use for padding + * + * This function copies @len - 1 bytes of string data from a userspace + * source pointer to a buffer backend, followed by a terminating '\0' + * character, at the current context offset. This is more or less a + * buffer backend-specific strncpy() operation. If a terminating '\0' + * character is found in @src before @len - 1 characters are copied, pad + * the buffer with @pad characters (e.g. '#'). Calls the slow path + * (_ring_buffer_strcpy_from_user_inatomic) if copy is crossing a page + * boundary. Disable the page fault handler to ensure we never try to + * take the mmap_sem. + */ +static inline +void lib_ring_buffer_strcpy_from_user_inatomic(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx, + const void __user *src, size_t len, int pad) +{ + struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; + struct channel_backend *chanb = &ctx->chan->backend; + size_t index, pagecpy; + size_t offset = ctx->buf_offset; + struct lib_ring_buffer_backend_pages *backend_pages; + mm_segment_t old_fs = get_fs(); + + if (unlikely(!len)) + return; + backend_pages = + lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); + offset &= chanb->buf_size - 1; + index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; + pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); + + set_fs(KERNEL_DS); + pagefault_disable(); + if (unlikely(!access_ok(src, len))) + goto fill_buffer; + + if (likely(pagecpy == len)) { + size_t count; + + count = lib_ring_buffer_do_strcpy_from_user_inatomic(config, + backend_pages->p[index].virt + + (offset & ~PAGE_MASK), + src, len - 1); + offset += count; + /* Padding */ + if (unlikely(count < len - 1)) { + size_t pad_len = len - 1 - count; + + lib_ring_buffer_do_memset(backend_pages->p[index].virt + + (offset & ~PAGE_MASK), + pad, pad_len); + offset += pad_len; + } + /* Ending '\0' */ + lib_ring_buffer_do_memset(backend_pages->p[index].virt + + (offset & ~PAGE_MASK), + '\0', 1); + } else { + _lib_ring_buffer_strcpy_from_user_inatomic(bufb, offset, src, + len, 0, pad); + } + pagefault_enable(); + set_fs(old_fs); + ctx->buf_offset += len; + + return; + +fill_buffer: + pagefault_enable(); + set_fs(old_fs); + /* + * In the error path we call the slow path version to avoid + * the pollution of static inline code. + */ + _lib_ring_buffer_memset(bufb, offset, pad, len - 1, 0); + offset += len - 1; + _lib_ring_buffer_memset(bufb, offset, '\0', 1, 0); +} + +/* + * This accessor counts the number of unread records in a buffer. + * It only provides a consistent value if no reads not writes are performed + * concurrently. + */ +static inline +unsigned long lib_ring_buffer_get_records_unread( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + struct lib_ring_buffer_backend *bufb = &buf->backend; + struct lib_ring_buffer_backend_pages *pages; + unsigned long records_unread = 0, sb_bindex, id; + unsigned int i; + + for (i = 0; i < bufb->chan->backend.num_subbuf; i++) { + id = bufb->buf_wsb[i].id; + sb_bindex = subbuffer_id_get_index(config, id); + pages = bufb->array[sb_bindex]; + records_unread += v_read(config, &pages->records_unread); + } + if (config->mode == RING_BUFFER_OVERWRITE) { + id = bufb->buf_rsb.id; + sb_bindex = subbuffer_id_get_index(config, id); + pages = bufb->array[sb_bindex]; + records_unread += v_read(config, &pages->records_unread); + } + return records_unread; +} + +/* + * We use __copy_from_user_inatomic to copy userspace data after + * checking with access_ok() and disabling page faults. + * + * Return 0 if OK, nonzero on error. + */ +static inline +unsigned long lib_ring_buffer_copy_from_user_check_nofault(void *dest, + const void __user *src, + unsigned long len) +{ + unsigned long ret; + mm_segment_t old_fs; + + if (!access_ok(src, len)) + return 1; + old_fs = get_fs(); + set_fs(KERNEL_DS); + pagefault_disable(); + ret = __copy_from_user_inatomic(dest, src, len); + pagefault_enable(); + set_fs(old_fs); + return ret; +} + +#endif /* _LIB_RING_BUFFER_BACKEND_H */ diff --git a/include/ringbuffer/backend_internal.h b/include/ringbuffer/backend_internal.h index 3bb1586e..895dbd9b 100644 --- a/include/ringbuffer/backend_internal.h +++ b/include/ringbuffer/backend_internal.h @@ -1,2 +1,531 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/backend_internal.h + * + * Ring buffer backend (internal helpers). + * + * Copyright (C) 2008-2012 Mathieu Desnoyers + */ + +#ifndef _LIB_RING_BUFFER_BACKEND_INTERNAL_H +#define _LIB_RING_BUFFER_BACKEND_INTERNAL_H + +#include +#include +#include +#include +#include + +/* Ring buffer backend API presented to the frontend */ + +/* Ring buffer and channel backend create/free */ + +int lib_ring_buffer_backend_create(struct lib_ring_buffer_backend *bufb, + struct channel_backend *chan, int cpu); +void channel_backend_unregister_notifiers(struct channel_backend *chanb); +void lib_ring_buffer_backend_free(struct lib_ring_buffer_backend *bufb); +int channel_backend_init(struct channel_backend *chanb, + const char *name, + const struct lib_ring_buffer_config *config, + void *priv, size_t subbuf_size, + size_t num_subbuf); +void channel_backend_free(struct channel_backend *chanb); + +void lib_ring_buffer_backend_reset(struct lib_ring_buffer_backend *bufb); +void channel_backend_reset(struct channel_backend *chanb); + +int lib_ring_buffer_backend_init(void); +void lib_ring_buffer_backend_exit(void); + +extern void _lib_ring_buffer_write(struct lib_ring_buffer_backend *bufb, + size_t offset, const void *src, size_t len, + size_t pagecpy); +extern void _lib_ring_buffer_memset(struct lib_ring_buffer_backend *bufb, + size_t offset, int c, size_t len, + size_t pagecpy); +extern void _lib_ring_buffer_strcpy(struct lib_ring_buffer_backend *bufb, + size_t offset, const char *src, size_t len, + size_t pagecpy, int pad); +extern void _lib_ring_buffer_copy_from_user_inatomic(struct lib_ring_buffer_backend *bufb, + size_t offset, const void *src, + size_t len, size_t pagecpy); +extern void _lib_ring_buffer_strcpy_from_user_inatomic(struct lib_ring_buffer_backend *bufb, + size_t offset, const char __user *src, size_t len, + size_t pagecpy, int pad); + +/* + * Subbuffer ID bits for overwrite mode. Need to fit within a single word to be + * exchanged atomically. + * + * Top half word, except lowest bit, belongs to "offset", which is used to keep + * to count the produced buffers. For overwrite mode, this provides the + * consumer with the capacity to read subbuffers in order, handling the + * situation where producers would write up to 2^15 buffers (or 2^31 for 64-bit + * systems) concurrently with a single execution of get_subbuf (between offset + * sampling and subbuffer ID exchange). + */ + +#define HALF_ULONG_BITS (BITS_PER_LONG >> 1) + +#define SB_ID_OFFSET_SHIFT (HALF_ULONG_BITS + 1) +#define SB_ID_OFFSET_COUNT (1UL << SB_ID_OFFSET_SHIFT) +#define SB_ID_OFFSET_MASK (~(SB_ID_OFFSET_COUNT - 1)) +/* + * Lowest bit of top word half belongs to noref. Used only for overwrite mode. + */ +#define SB_ID_NOREF_SHIFT (SB_ID_OFFSET_SHIFT - 1) +#define SB_ID_NOREF_COUNT (1UL << SB_ID_NOREF_SHIFT) +#define SB_ID_NOREF_MASK SB_ID_NOREF_COUNT +/* + * In overwrite mode: lowest half of word is used for index. + * Limit of 2^16 subbuffers per buffer on 32-bit, 2^32 on 64-bit. + * In producer-consumer mode: whole word used for index. + */ +#define SB_ID_INDEX_SHIFT 0 +#define SB_ID_INDEX_COUNT (1UL << SB_ID_INDEX_SHIFT) +#define SB_ID_INDEX_MASK (SB_ID_NOREF_COUNT - 1) + +/* + * Construct the subbuffer id from offset, index and noref. Use only the index + * for producer-consumer mode (offset and noref are only used in overwrite + * mode). + */ +static inline +unsigned long subbuffer_id(const struct lib_ring_buffer_config *config, + unsigned long offset, unsigned long noref, + unsigned long index) +{ + if (config->mode == RING_BUFFER_OVERWRITE) + return (offset << SB_ID_OFFSET_SHIFT) + | (noref << SB_ID_NOREF_SHIFT) + | index; + else + return index; +} + +/* + * Compare offset with the offset contained within id. Return 1 if the offset + * bits are identical, else 0. + */ +static inline +int subbuffer_id_compare_offset(const struct lib_ring_buffer_config *config, + unsigned long id, unsigned long offset) +{ + return (id & SB_ID_OFFSET_MASK) == (offset << SB_ID_OFFSET_SHIFT); +} + +static inline +unsigned long subbuffer_id_get_index(const struct lib_ring_buffer_config *config, + unsigned long id) +{ + if (config->mode == RING_BUFFER_OVERWRITE) + return id & SB_ID_INDEX_MASK; + else + return id; +} + +static inline +unsigned long subbuffer_id_is_noref(const struct lib_ring_buffer_config *config, + unsigned long id) +{ + if (config->mode == RING_BUFFER_OVERWRITE) + return !!(id & SB_ID_NOREF_MASK); + else + return 1; +} + +/* + * Only used by reader on subbuffer ID it has exclusive access to. No volatile + * needed. + */ +static inline +void subbuffer_id_set_noref(const struct lib_ring_buffer_config *config, + unsigned long *id) +{ + if (config->mode == RING_BUFFER_OVERWRITE) + *id |= SB_ID_NOREF_MASK; +} + +static inline +void subbuffer_id_set_noref_offset(const struct lib_ring_buffer_config *config, + unsigned long *id, unsigned long offset) +{ + unsigned long tmp; + + if (config->mode == RING_BUFFER_OVERWRITE) { + tmp = *id; + tmp &= ~SB_ID_OFFSET_MASK; + tmp |= offset << SB_ID_OFFSET_SHIFT; + tmp |= SB_ID_NOREF_MASK; + /* Volatile store, read concurrently by readers. */ + WRITE_ONCE(*id, tmp); + } +} + +/* No volatile access, since already used locally */ +static inline +void subbuffer_id_clear_noref(const struct lib_ring_buffer_config *config, + unsigned long *id) +{ + if (config->mode == RING_BUFFER_OVERWRITE) + *id &= ~SB_ID_NOREF_MASK; +} + +/* + * For overwrite mode, cap the number of subbuffers per buffer to: + * 2^16 on 32-bit architectures + * 2^32 on 64-bit architectures + * This is required to fit in the index part of the ID. Return 0 on success, + * -EPERM on failure. + */ +static inline +int subbuffer_id_check_index(const struct lib_ring_buffer_config *config, + unsigned long num_subbuf) +{ + if (config->mode == RING_BUFFER_OVERWRITE) + return (num_subbuf > (1UL << HALF_ULONG_BITS)) ? -EPERM : 0; + else + return 0; +} + +static inline +void lib_ring_buffer_backend_get_pages(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx, + struct lib_ring_buffer_backend_pages **backend_pages) +{ + struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; + struct channel_backend *chanb = &ctx->chan->backend; + size_t sbidx, offset = ctx->buf_offset; + unsigned long sb_bindex, id; + struct lib_ring_buffer_backend_pages *rpages; + + offset &= chanb->buf_size - 1; + sbidx = offset >> chanb->subbuf_size_order; + id = bufb->buf_wsb[sbidx].id; + sb_bindex = subbuffer_id_get_index(config, id); + rpages = bufb->array[sb_bindex]; + CHAN_WARN_ON(ctx->chan, + config->mode == RING_BUFFER_OVERWRITE + && subbuffer_id_is_noref(config, id)); + *backend_pages = rpages; +} + +/* Get backend pages from cache. */ +static inline +struct lib_ring_buffer_backend_pages * + lib_ring_buffer_get_backend_pages_from_ctx(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx) +{ + return ctx->backend_pages; +} + +/* + * The ring buffer can count events recorded and overwritten per buffer, + * but it is disabled by default due to its performance overhead. + */ +#ifdef LTTNG_RING_BUFFER_COUNT_EVENTS +static inline +void subbuffer_count_record(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + unsigned long idx) +{ + unsigned long sb_bindex; + + sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); + v_inc(config, &bufb->array[sb_bindex]->records_commit); +} +#else /* LTTNG_RING_BUFFER_COUNT_EVENTS */ +static inline +void subbuffer_count_record(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + unsigned long idx) +{ +} +#endif /* #else LTTNG_RING_BUFFER_COUNT_EVENTS */ + +/* + * Reader has exclusive subbuffer access for record consumption. No need to + * perform the decrement atomically. + */ +static inline +void subbuffer_consume_record(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb) +{ + unsigned long sb_bindex; + + sb_bindex = subbuffer_id_get_index(config, bufb->buf_rsb.id); + CHAN_WARN_ON(bufb->chan, + !v_read(config, &bufb->array[sb_bindex]->records_unread)); + /* Non-atomic decrement protected by exclusive subbuffer access */ + _v_dec(config, &bufb->array[sb_bindex]->records_unread); + v_inc(config, &bufb->records_read); +} + +static inline +unsigned long subbuffer_get_records_count( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + unsigned long idx) +{ + unsigned long sb_bindex; + + sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); + return v_read(config, &bufb->array[sb_bindex]->records_commit); +} + +/* + * Must be executed at subbuffer delivery when the writer has _exclusive_ + * subbuffer access. See lib_ring_buffer_check_deliver() for details. + * lib_ring_buffer_get_records_count() must be called to get the records + * count before this function, because it resets the records_commit + * count. + */ +static inline +unsigned long subbuffer_count_records_overrun( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + unsigned long idx) +{ + struct lib_ring_buffer_backend_pages *pages; + unsigned long overruns, sb_bindex; + + sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); + pages = bufb->array[sb_bindex]; + overruns = v_read(config, &pages->records_unread); + v_set(config, &pages->records_unread, + v_read(config, &pages->records_commit)); + v_set(config, &pages->records_commit, 0); + + return overruns; +} + +static inline +void subbuffer_set_data_size(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + unsigned long idx, + unsigned long data_size) +{ + struct lib_ring_buffer_backend_pages *pages; + unsigned long sb_bindex; + + sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); + pages = bufb->array[sb_bindex]; + pages->data_size = data_size; +} + +static inline +unsigned long subbuffer_get_read_data_size( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb) +{ + struct lib_ring_buffer_backend_pages *pages; + unsigned long sb_bindex; + + sb_bindex = subbuffer_id_get_index(config, bufb->buf_rsb.id); + pages = bufb->array[sb_bindex]; + return pages->data_size; +} + +static inline +unsigned long subbuffer_get_data_size( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + unsigned long idx) +{ + struct lib_ring_buffer_backend_pages *pages; + unsigned long sb_bindex; + + sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); + pages = bufb->array[sb_bindex]; + return pages->data_size; +} + +static inline +void subbuffer_inc_packet_count(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + unsigned long idx) +{ + bufb->buf_cnt[idx].seq_cnt++; +} + +/** + * lib_ring_buffer_clear_noref - Clear the noref subbuffer flag, called by + * writer. + */ +static inline +void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + unsigned long idx) +{ + unsigned long id, new_id; + + if (config->mode != RING_BUFFER_OVERWRITE) + return; + + /* + * Performing a volatile access to read the sb_pages, because we want to + * read a coherent version of the pointer and the associated noref flag. + */ + id = READ_ONCE(bufb->buf_wsb[idx].id); + for (;;) { + /* This check is called on the fast path for each record. */ + if (likely(!subbuffer_id_is_noref(config, id))) { + /* + * Store after load dependency ordering the writes to + * the subbuffer after load and test of the noref flag + * matches the memory barrier implied by the cmpxchg() + * in update_read_sb_index(). + */ + return; /* Already writing to this buffer */ + } + new_id = id; + subbuffer_id_clear_noref(config, &new_id); + new_id = cmpxchg(&bufb->buf_wsb[idx].id, id, new_id); + if (likely(new_id == id)) + break; + id = new_id; + } +} + +/** + * lib_ring_buffer_set_noref_offset - Set the noref subbuffer flag and offset, + * called by writer. + */ +static inline +void lib_ring_buffer_set_noref_offset(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + unsigned long idx, unsigned long offset) +{ + if (config->mode != RING_BUFFER_OVERWRITE) + return; + + /* + * Because ring_buffer_set_noref() is only called by a single thread + * (the one which updated the cc_sb value), there are no concurrent + * updates to take care of: other writers have not updated cc_sb, so + * they cannot set the noref flag, and concurrent readers cannot modify + * the pointer because the noref flag is not set yet. + * The smp_wmb() in ring_buffer_commit() takes care of ordering writes + * to the subbuffer before this set noref operation. + * subbuffer_set_noref() uses a volatile store to deal with concurrent + * readers of the noref flag. + */ + CHAN_WARN_ON(bufb->chan, + subbuffer_id_is_noref(config, bufb->buf_wsb[idx].id)); + /* + * Memory barrier that ensures counter stores are ordered before set + * noref and offset. + */ + smp_mb(); + subbuffer_id_set_noref_offset(config, &bufb->buf_wsb[idx].id, offset); +} + +/** + * update_read_sb_index - Read-side subbuffer index update. + */ +static inline +int update_read_sb_index(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_backend *bufb, + struct channel_backend *chanb, + unsigned long consumed_idx, + unsigned long consumed_count) +{ + unsigned long old_id, new_id; + + if (config->mode == RING_BUFFER_OVERWRITE) { + /* + * Exchange the target writer subbuffer with our own unused + * subbuffer. No need to use READ_ONCE() here to read the + * old_wpage, because the value read will be confirmed by the + * following cmpxchg(). + */ + old_id = bufb->buf_wsb[consumed_idx].id; + if (unlikely(!subbuffer_id_is_noref(config, old_id))) + return -EAGAIN; + /* + * Make sure the offset count we are expecting matches the one + * indicated by the writer. + */ + if (unlikely(!subbuffer_id_compare_offset(config, old_id, + consumed_count))) + return -EAGAIN; + CHAN_WARN_ON(bufb->chan, + !subbuffer_id_is_noref(config, bufb->buf_rsb.id)); + subbuffer_id_set_noref_offset(config, &bufb->buf_rsb.id, + consumed_count); + new_id = cmpxchg(&bufb->buf_wsb[consumed_idx].id, old_id, + bufb->buf_rsb.id); + if (unlikely(old_id != new_id)) + return -EAGAIN; + bufb->buf_rsb.id = new_id; + } else { + /* No page exchange, use the writer page directly */ + bufb->buf_rsb.id = bufb->buf_wsb[consumed_idx].id; + } + return 0; +} + +static inline __attribute__((always_inline)) +void lttng_inline_memcpy(void *dest, const void *src, + unsigned long len) +{ + switch (len) { + case 1: + *(uint8_t *) dest = *(const uint8_t *) src; + break; + case 2: + *(uint16_t *) dest = *(const uint16_t *) src; + break; + case 4: + *(uint32_t *) dest = *(const uint32_t *) src; + break; + case 8: + *(uint64_t *) dest = *(const uint64_t *) src; + break; + default: + memcpy(dest, src, len); + } +} + +/* + * Use the architecture-specific memcpy implementation for constant-sized + * inputs, but rely on an inline memcpy for length statically unknown. + * The function call to memcpy is just way too expensive for a fast path. + */ +#define lib_ring_buffer_do_copy(config, dest, src, len) \ +do { \ + size_t __len = (len); \ + if (__builtin_constant_p(len)) \ + memcpy(dest, src, __len); \ + else \ + lttng_inline_memcpy(dest, src, __len); \ +} while (0) + +/* + * We use __copy_from_user_inatomic to copy userspace data since we already + * did the access_ok for the whole range. + * + * Return 0 if OK, nonzero on error. + */ +static inline +unsigned long lib_ring_buffer_do_copy_from_user_inatomic(void *dest, + const void __user *src, + unsigned long len) +{ + return __copy_from_user_inatomic(dest, src, len); +} + +/* + * write len bytes to dest with c + */ +static inline +void lib_ring_buffer_do_memset(char *dest, int c, + unsigned long len) +{ + unsigned long i; + + for (i = 0; i < len; i++) + dest[i] = c; +} + +#endif /* _LIB_RING_BUFFER_BACKEND_INTERNAL_H */ diff --git a/include/ringbuffer/backend_types.h b/include/ringbuffer/backend_types.h index ff237c03..f1b3ee27 100644 --- a/include/ringbuffer/backend_types.h +++ b/include/ringbuffer/backend_types.h @@ -1,2 +1,100 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/backend_types.h + * + * Ring buffer backend (types). + * + * Copyright (C) 2008-2012 Mathieu Desnoyers + */ + +#ifndef _LIB_RING_BUFFER_BACKEND_TYPES_H +#define _LIB_RING_BUFFER_BACKEND_TYPES_H + +#include +#include +#include +#include + +struct lib_ring_buffer_backend_page { + void *virt; /* page virtual address (cached) */ + unsigned long pfn; /* page frame number */ +}; + +struct lib_ring_buffer_backend_pages { + unsigned long mmap_offset; /* offset of the subbuffer in mmap */ + union v_atomic records_commit; /* current records committed count */ + union v_atomic records_unread; /* records to read */ + unsigned long data_size; /* Amount of data to read from subbuf */ + struct lib_ring_buffer_backend_page p[]; +}; + +struct lib_ring_buffer_backend_subbuffer { + /* Identifier for subbuf backend pages. Exchanged atomically. */ + unsigned long id; /* backend subbuffer identifier */ +}; + +struct lib_ring_buffer_backend_counts { + /* + * Counter specific to the sub-buffer location within the ring buffer. + * The actual sequence number of the packet within the entire ring + * buffer can be derived from the formula nr_subbuffers * seq_cnt + + * subbuf_idx. + */ + uint64_t seq_cnt; /* packet sequence number */ +}; + +/* + * Forward declaration of frontend-specific channel and ring_buffer. + */ +struct channel; +struct lib_ring_buffer; + +struct lib_ring_buffer_backend { + /* Array of ring_buffer_backend_subbuffer for writer */ + struct lib_ring_buffer_backend_subbuffer *buf_wsb; + /* ring_buffer_backend_subbuffer for reader */ + struct lib_ring_buffer_backend_subbuffer buf_rsb; + /* Array of lib_ring_buffer_backend_counts for the packet counter */ + struct lib_ring_buffer_backend_counts *buf_cnt; + /* + * Pointer array of backend pages, for whole buffer. + * Indexed by ring_buffer_backend_subbuffer identifier (id) index. + */ + struct lib_ring_buffer_backend_pages **array; + unsigned int num_pages_per_subbuf; + + struct channel *chan; /* Associated channel */ + int cpu; /* This buffer's cpu. -1 if global. */ + union v_atomic records_read; /* Number of records read */ + unsigned int allocated:1; /* is buffer allocated ? */ +}; + +struct channel_backend { + unsigned long buf_size; /* Size of the buffer */ + unsigned long subbuf_size; /* Sub-buffer size */ + unsigned int subbuf_size_order; /* Order of sub-buffer size */ + unsigned int num_subbuf_order; /* + * Order of number of sub-buffers/buffer + * for writer. + */ + unsigned int buf_size_order; /* Order of buffer size */ + unsigned int extra_reader_sb:1; /* has extra reader subbuffer ? */ + struct lib_ring_buffer *buf; /* Channel per-cpu buffers */ + + unsigned long num_subbuf; /* Number of sub-buffers for writer */ + u64 start_tsc; /* Channel creation TSC value */ + void *priv; /* Client-specific information */ + void *priv_ops; /* Client-specific ops pointer */ + void (*release_priv_ops)(void *priv_ops); + struct lttng_cpuhp_node cpuhp_prepare; /* CPU hotplug prepare */ + /* + * We need to copy config because the module containing the + * source config can vanish before the last reference to this + * channel's streams is released. + */ + struct lib_ring_buffer_config config; /* Ring buffer configuration */ + cpumask_var_t cpumask; /* Allocated per-cpu buffers cpumask */ + char name[NAME_MAX]; /* Channel name */ +}; + +#endif /* _LIB_RING_BUFFER_BACKEND_TYPES_H */ diff --git a/include/ringbuffer/config.h b/include/ringbuffer/config.h index e1f0a965..df7c335d 100644 --- a/include/ringbuffer/config.h +++ b/include/ringbuffer/config.h @@ -1,2 +1,306 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/config.h + * + * Ring buffer configuration header. Note: after declaring the standard inline + * functions, clients should also include linux/ringbuffer/api.h. + * + * Copyright (C) 2010-2012 Mathieu Desnoyers + */ + +#ifndef _LIB_RING_BUFFER_CONFIG_H +#define _LIB_RING_BUFFER_CONFIG_H + +#include +#include +#include +#include + +struct lib_ring_buffer; +struct channel; +struct lib_ring_buffer_config; +struct lib_ring_buffer_ctx; + +/* + * Ring buffer client callbacks. Only used by slow path, never on fast path. + * For the fast path, record_header_size(), ring_buffer_clock_read() should be + * provided as inline functions too. These may simply return 0 if not used by + * the client. + */ +struct lib_ring_buffer_client_cb { + /* Mandatory callbacks */ + + /* A static inline version is also required for fast path */ + u64 (*ring_buffer_clock_read) (struct channel *chan); + size_t (*record_header_size) (const struct lib_ring_buffer_config *config, + struct channel *chan, size_t offset, + size_t *pre_header_padding, + struct lib_ring_buffer_ctx *ctx, + void *client_ctx); + + /* Slow path only, at subbuffer switch */ + size_t (*subbuffer_header_size) (void); + void (*buffer_begin) (struct lib_ring_buffer *buf, u64 tsc, + unsigned int subbuf_idx); + void (*buffer_end) (struct lib_ring_buffer *buf, u64 tsc, + unsigned int subbuf_idx, unsigned long data_size); + + /* Optional callbacks (can be set to NULL) */ + + /* Called at buffer creation/finalize */ + int (*buffer_create) (struct lib_ring_buffer *buf, void *priv, + int cpu, const char *name); + /* + * Clients should guarantee that no new reader handle can be opened + * after finalize. + */ + void (*buffer_finalize) (struct lib_ring_buffer *buf, void *priv, int cpu); + + /* + * Extract header length, payload length and timestamp from event + * record. Used by buffer iterators. Timestamp is only used by channel + * iterator. + */ + void (*record_get) (const struct lib_ring_buffer_config *config, + struct channel *chan, struct lib_ring_buffer *buf, + size_t offset, size_t *header_len, + size_t *payload_len, u64 *timestamp); +}; + +/* + * Ring buffer instance configuration. + * + * Declare as "static const" within the client object to ensure the inline fast + * paths can be optimized. + * + * alloc/sync pairs: + * + * RING_BUFFER_ALLOC_PER_CPU and RING_BUFFER_SYNC_PER_CPU : + * Per-cpu buffers with per-cpu synchronization. Tracing must be performed + * with preemption disabled (lib_ring_buffer_get_cpu() and + * lib_ring_buffer_put_cpu()). + * + * RING_BUFFER_ALLOC_PER_CPU and RING_BUFFER_SYNC_GLOBAL : + * Per-cpu buffer with global synchronization. Tracing can be performed with + * preemption enabled, statistically stays on the local buffers. + * + * RING_BUFFER_ALLOC_GLOBAL and RING_BUFFER_SYNC_PER_CPU : + * Should only be used for buffers belonging to a single thread or protected + * by mutual exclusion by the client. Note that periodical sub-buffer switch + * should be disabled in this kind of configuration. + * + * RING_BUFFER_ALLOC_GLOBAL and RING_BUFFER_SYNC_GLOBAL : + * Global shared buffer with global synchronization. + * + * wakeup: + * + * RING_BUFFER_WAKEUP_BY_TIMER uses per-cpu timers to poll the + * buffers and wake up readers if data is ready. Mainly useful for tracers which + * don't want to call into the wakeup code on the tracing path. Use in + * combination with "read_timer_interval" channel_create() argument. + * + * RING_BUFFER_WAKEUP_BY_WRITER directly wakes up readers when a subbuffer is + * ready to read. Lower latencies before the reader is woken up. Mainly suitable + * for drivers. + * + * RING_BUFFER_WAKEUP_NONE does not perform any wakeup whatsoever. The client + * has the responsibility to perform wakeups. + */ +struct lib_ring_buffer_config { + enum { + RING_BUFFER_ALLOC_PER_CPU, + RING_BUFFER_ALLOC_GLOBAL, + } alloc; + enum { + RING_BUFFER_SYNC_PER_CPU, /* Wait-free */ + RING_BUFFER_SYNC_GLOBAL, /* Lock-free */ + } sync; + enum { + RING_BUFFER_OVERWRITE, /* Overwrite when buffer full */ + RING_BUFFER_DISCARD, /* Discard when buffer full */ + } mode; + enum { + RING_BUFFER_SPLICE, + RING_BUFFER_MMAP, + RING_BUFFER_READ, /* TODO */ + RING_BUFFER_ITERATOR, + RING_BUFFER_NONE, + } output; + enum { + RING_BUFFER_PAGE, + RING_BUFFER_VMAP, /* TODO */ + RING_BUFFER_STATIC, /* TODO */ + } backend; + enum { + RING_BUFFER_NO_OOPS_CONSISTENCY, + RING_BUFFER_OOPS_CONSISTENCY, + } oops; + enum { + RING_BUFFER_IPI_BARRIER, + RING_BUFFER_NO_IPI_BARRIER, + } ipi; + enum { + RING_BUFFER_WAKEUP_BY_TIMER, /* wake up performed by timer */ + RING_BUFFER_WAKEUP_BY_WRITER, /* + * writer wakes up reader, + * not lock-free + * (takes spinlock). + */ + } wakeup; + /* + * tsc_bits: timestamp bits saved at each record. + * 0 and 64 disable the timestamp compression scheme. + */ + unsigned int tsc_bits; + struct lib_ring_buffer_client_cb cb; +}; + +/* + * ring buffer context + * + * Context passed to lib_ring_buffer_reserve(), lib_ring_buffer_commit(), + * lib_ring_buffer_try_discard_reserve(), lib_ring_buffer_align_ctx() and + * lib_ring_buffer_write(). + */ +struct lib_ring_buffer_ctx { + /* input received by lib_ring_buffer_reserve(), saved here. */ + struct channel *chan; /* channel */ + void *priv; /* client private data */ + size_t data_size; /* size of payload */ + int largest_align; /* + * alignment of the largest element + * in the payload + */ + int cpu; /* processor id */ + + /* output from lib_ring_buffer_reserve() */ + struct lib_ring_buffer *buf; /* + * buffer corresponding to processor id + * for this channel + */ + size_t slot_size; /* size of the reserved slot */ + unsigned long buf_offset; /* offset following the record header */ + unsigned long pre_offset; /* + * Initial offset position _before_ + * the record is written. Positioned + * prior to record header alignment + * padding. + */ + u64 tsc; /* time-stamp counter value */ + unsigned int rflags; /* reservation flags */ + /* Cache backend pages pointer chasing. */ + struct lib_ring_buffer_backend_pages *backend_pages; +}; + +/** + * lib_ring_buffer_ctx_init - initialize ring buffer context + * @ctx: ring buffer context to initialize + * @chan: channel + * @priv: client private data + * @data_size: size of record data payload. It must be greater than 0. + * @largest_align: largest alignment within data payload types + * @cpu: processor id + */ +static inline +void lib_ring_buffer_ctx_init(struct lib_ring_buffer_ctx *ctx, + struct channel *chan, void *priv, + size_t data_size, int largest_align, + int cpu) +{ + ctx->chan = chan; + ctx->priv = priv; + ctx->data_size = data_size; + ctx->largest_align = largest_align; + ctx->cpu = cpu; + ctx->rflags = 0; + ctx->backend_pages = NULL; +} + +/* + * Reservation flags. + * + * RING_BUFFER_RFLAG_FULL_TSC + * + * This flag is passed to record_header_size() and to the primitive used to + * write the record header. It indicates that the full 64-bit time value is + * needed in the record header. If this flag is not set, the record header needs + * only to contain "tsc_bits" bit of time value. + * + * Reservation flags can be added by the client, starting from + * "(RING_BUFFER_FLAGS_END << 0)". It can be used to pass information from + * record_header_size() to lib_ring_buffer_write_record_header(). + */ +#define RING_BUFFER_RFLAG_FULL_TSC (1U << 0) +#define RING_BUFFER_RFLAG_END (1U << 1) + +#ifndef LTTNG_TRACER_CORE_H +#error "lttng-tracer-core.h is needed for RING_BUFFER_ALIGN define" +#endif + +/* + * We need to define RING_BUFFER_ALIGN_ATTR so it is known early at + * compile-time. We have to duplicate the "config->align" information and the + * definition here because config->align is used both in the slow and fast + * paths, but RING_BUFFER_ALIGN_ATTR is only available for the client code. + */ +#ifdef RING_BUFFER_ALIGN + +# define RING_BUFFER_ALIGN_ATTR /* Default arch alignment */ + +/* + * Calculate the offset needed to align the type. + * size_of_type must be non-zero. + */ +static inline +unsigned int lib_ring_buffer_align(size_t align_drift, size_t size_of_type) +{ + return offset_align(align_drift, size_of_type); +} + +#else + +# define RING_BUFFER_ALIGN_ATTR __attribute__((packed)) + +/* + * Calculate the offset needed to align the type. + * size_of_type must be non-zero. + */ +static inline +unsigned int lib_ring_buffer_align(size_t align_drift, size_t size_of_type) +{ + return 0; +} + +#endif + +/** + * lib_ring_buffer_align_ctx - Align context offset on "alignment" + * @ctx: ring buffer context. + */ +static inline +void lib_ring_buffer_align_ctx(struct lib_ring_buffer_ctx *ctx, + size_t alignment) +{ + ctx->buf_offset += lib_ring_buffer_align(ctx->buf_offset, + alignment); +} + +/* + * lib_ring_buffer_check_config() returns 0 on success. + * Used internally to check for valid configurations at channel creation. + */ +static inline +int lib_ring_buffer_check_config(const struct lib_ring_buffer_config *config, + unsigned int switch_timer_interval, + unsigned int read_timer_interval) +{ + if (config->alloc == RING_BUFFER_ALLOC_GLOBAL + && config->sync == RING_BUFFER_SYNC_PER_CPU + && switch_timer_interval) + return -EINVAL; + return 0; +} + +#include + +#endif /* _LIB_RING_BUFFER_CONFIG_H */ diff --git a/include/ringbuffer/frontend.h b/include/ringbuffer/frontend.h index 35421581..a15b658b 100644 --- a/include/ringbuffer/frontend.h +++ b/include/ringbuffer/frontend.h @@ -1,2 +1,234 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/frontend.h + * + * Ring Buffer Library Synchronization Header (API). + * + * Copyright (C) 2005-2012 Mathieu Desnoyers + * + * See ring_buffer_frontend.c for more information on wait-free algorithms. + */ + +#ifndef _LIB_RING_BUFFER_FRONTEND_H +#define _LIB_RING_BUFFER_FRONTEND_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* Internal helpers */ +#include + +/* Max ring buffer nesting count, see lib_ring_buffer_get_cpu(). */ +#define RING_BUFFER_MAX_NESTING 4 + +/* Buffer creation/removal and setup operations */ + +/* + * switch_timer_interval is the time interval (in us) to fill sub-buffers with + * padding to let readers get those sub-buffers. Used for live streaming. + * + * read_timer_interval is the time interval (in us) to wake up pending readers. + * + * buf_addr is a pointer the the beginning of the preallocated buffer contiguous + * address mapping. It is used only by RING_BUFFER_STATIC configuration. It can + * be set to NULL for other backends. + */ + +extern +struct channel *channel_create(const struct lib_ring_buffer_config *config, + const char *name, void *priv, + void *buf_addr, + size_t subbuf_size, size_t num_subbuf, + unsigned int switch_timer_interval, + unsigned int read_timer_interval); + +/* + * channel_destroy returns the private data pointer. It finalizes all channel's + * buffers, waits for readers to release all references, and destroys the + * channel. + */ +extern +void *channel_destroy(struct channel *chan); + + +/* Buffer read operations */ + +/* + * Iteration on channel cpumask needs to issue a read barrier to match the write + * barrier in cpu hotplug. It orders the cpumask read before read of per-cpu + * buffer data. The per-cpu buffer is never removed by cpu hotplug; teardown is + * only performed at channel destruction. + */ +#define for_each_channel_cpu(cpu, chan) \ + for ((cpu) = -1; \ + ({ (cpu) = cpumask_next(cpu, (chan)->backend.cpumask); \ + smp_read_barrier_depends(); (cpu) < nr_cpu_ids; });) + +extern struct lib_ring_buffer *channel_get_ring_buffer( + const struct lib_ring_buffer_config *config, + struct channel *chan, int cpu); +extern int lib_ring_buffer_open_read(struct lib_ring_buffer *buf); +extern void lib_ring_buffer_release_read(struct lib_ring_buffer *buf); + +/* + * Read sequence: snapshot, many get_subbuf/put_subbuf, move_consumer. + */ +extern int lib_ring_buffer_snapshot(struct lib_ring_buffer *buf, + unsigned long *consumed, + unsigned long *produced); +extern int lib_ring_buffer_snapshot_sample_positions( + struct lib_ring_buffer *buf, + unsigned long *consumed, + unsigned long *produced); +extern void lib_ring_buffer_move_consumer(struct lib_ring_buffer *buf, + unsigned long consumed_new); + +extern int lib_ring_buffer_get_subbuf(struct lib_ring_buffer *buf, + unsigned long consumed); +extern void lib_ring_buffer_put_subbuf(struct lib_ring_buffer *buf); + +void lib_ring_buffer_set_quiescent_channel(struct channel *chan); +void lib_ring_buffer_clear_quiescent_channel(struct channel *chan); + +/* + * lib_ring_buffer_get_next_subbuf/lib_ring_buffer_put_next_subbuf are helpers + * to read sub-buffers sequentially. + */ +static inline int lib_ring_buffer_get_next_subbuf(struct lib_ring_buffer *buf) +{ + int ret; + + ret = lib_ring_buffer_snapshot(buf, &buf->cons_snapshot, + &buf->prod_snapshot); + if (ret) + return ret; + ret = lib_ring_buffer_get_subbuf(buf, buf->cons_snapshot); + return ret; +} + +static inline void lib_ring_buffer_put_next_subbuf(struct lib_ring_buffer *buf) +{ + lib_ring_buffer_put_subbuf(buf); + lib_ring_buffer_move_consumer(buf, subbuf_align(buf->cons_snapshot, + buf->backend.chan)); +} + +extern void channel_reset(struct channel *chan); +extern void lib_ring_buffer_reset(struct lib_ring_buffer *buf); + +static inline +unsigned long lib_ring_buffer_get_offset(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + return v_read(config, &buf->offset); +} + +static inline +unsigned long lib_ring_buffer_get_consumed(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + return atomic_long_read(&buf->consumed); +} + +/* + * Must call lib_ring_buffer_is_finalized before reading counters (memory + * ordering enforced with respect to trace teardown). + */ +static inline +int lib_ring_buffer_is_finalized(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + int finalized = READ_ONCE(buf->finalized); + /* + * Read finalized before counters. + */ + smp_rmb(); + return finalized; +} + +static inline +int lib_ring_buffer_channel_is_finalized(const struct channel *chan) +{ + return chan->finalized; +} + +static inline +int lib_ring_buffer_channel_is_disabled(const struct channel *chan) +{ + return atomic_read(&chan->record_disabled); +} + +static inline +unsigned long lib_ring_buffer_get_read_data_size( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + return subbuffer_get_read_data_size(config, &buf->backend); +} + +static inline +unsigned long lib_ring_buffer_get_records_count( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + return v_read(config, &buf->records_count); +} + +static inline +unsigned long lib_ring_buffer_get_records_overrun( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + return v_read(config, &buf->records_overrun); +} + +static inline +unsigned long lib_ring_buffer_get_records_lost_full( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + return v_read(config, &buf->records_lost_full); +} + +static inline +unsigned long lib_ring_buffer_get_records_lost_wrap( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + return v_read(config, &buf->records_lost_wrap); +} + +static inline +unsigned long lib_ring_buffer_get_records_lost_big( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + return v_read(config, &buf->records_lost_big); +} + +static inline +unsigned long lib_ring_buffer_get_records_read( + const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + return v_read(config, &buf->backend.records_read); +} + +#endif /* _LIB_RING_BUFFER_FRONTEND_H */ diff --git a/include/ringbuffer/frontend_api.h b/include/ringbuffer/frontend_api.h index 53187bf5..bd43bfb7 100644 --- a/include/ringbuffer/frontend_api.h +++ b/include/ringbuffer/frontend_api.h @@ -1,2 +1,358 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/frontend_api.h + * + * Ring Buffer Library Synchronization Header (buffer write API). + * + * Copyright (C) 2005-2012 Mathieu Desnoyers + * + * See ring_buffer_frontend.c for more information on wait-free algorithms. + * See linux/ringbuffer/frontend.h for channel allocation and read-side API. + */ + +#ifndef _LIB_RING_BUFFER_FRONTEND_API_H +#define _LIB_RING_BUFFER_FRONTEND_API_H + +#include +#include +#include +#include + +/** + * lib_ring_buffer_get_cpu - Precedes ring buffer reserve/commit. + * + * Disables preemption (acts as a RCU read-side critical section) and keeps a + * ring buffer nesting count as supplementary safety net to ensure tracer client + * code will never trigger an endless recursion. Returns the processor ID on + * success, -EPERM on failure (nesting count too high). + * + * asm volatile and "memory" clobber prevent the compiler from moving + * instructions out of the ring buffer nesting count. This is required to ensure + * that probe side-effects which can cause recursion (e.g. unforeseen traps, + * divisions by 0, ...) are triggered within the incremented nesting count + * section. + */ +static inline +int lib_ring_buffer_get_cpu(const struct lib_ring_buffer_config *config) +{ + int cpu, nesting; + + rcu_read_lock_sched_notrace(); + cpu = smp_processor_id(); + nesting = ++per_cpu(lib_ring_buffer_nesting, cpu); + barrier(); + + if (unlikely(nesting > RING_BUFFER_MAX_NESTING)) { + WARN_ON_ONCE(1); + per_cpu(lib_ring_buffer_nesting, cpu)--; + rcu_read_unlock_sched_notrace(); + return -EPERM; + } else + return cpu; +} + +/** + * lib_ring_buffer_put_cpu - Follows ring buffer reserve/commit. + */ +static inline +void lib_ring_buffer_put_cpu(const struct lib_ring_buffer_config *config) +{ + barrier(); + (*this_cpu_ptr(&lib_ring_buffer_nesting))--; + rcu_read_unlock_sched_notrace(); +} + +/* + * lib_ring_buffer_try_reserve is called by lib_ring_buffer_reserve(). It is not + * part of the API per se. + * + * returns 0 if reserve ok, or 1 if the slow path must be taken. + */ +static inline +int lib_ring_buffer_try_reserve(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx, + void *client_ctx, + unsigned long *o_begin, unsigned long *o_end, + unsigned long *o_old, size_t *before_hdr_pad) +{ + struct channel *chan = ctx->chan; + struct lib_ring_buffer *buf = ctx->buf; + *o_begin = v_read(config, &buf->offset); + *o_old = *o_begin; + + ctx->tsc = lib_ring_buffer_clock_read(chan); + if ((int64_t) ctx->tsc == -EIO) + return 1; + + /* + * Prefetch cacheline for read because we have to read the previous + * commit counter to increment it and commit seq value to compare it to + * the commit counter. + */ + prefetch(&buf->commit_hot[subbuf_index(*o_begin, chan)]); + + if (last_tsc_overflow(config, buf, ctx->tsc)) + ctx->rflags |= RING_BUFFER_RFLAG_FULL_TSC; + + if (unlikely(subbuf_offset(*o_begin, chan) == 0)) + return 1; + + ctx->slot_size = record_header_size(config, chan, *o_begin, + before_hdr_pad, ctx, client_ctx); + ctx->slot_size += + lib_ring_buffer_align(*o_begin + ctx->slot_size, + ctx->largest_align) + ctx->data_size; + if (unlikely((subbuf_offset(*o_begin, chan) + ctx->slot_size) + > chan->backend.subbuf_size)) + return 1; + + /* + * Record fits in the current buffer and we are not on a switch + * boundary. It's safe to write. + */ + *o_end = *o_begin + ctx->slot_size; + + if (unlikely((subbuf_offset(*o_end, chan)) == 0)) + /* + * The offset_end will fall at the very beginning of the next + * subbuffer. + */ + return 1; + + return 0; +} + +/** + * lib_ring_buffer_reserve - Reserve space in a ring buffer. + * @config: ring buffer instance configuration. + * @ctx: ring buffer context. (input and output) Must be already initialized. + * + * Atomic wait-free slot reservation. The reserved space starts at the context + * "pre_offset". Its length is "slot_size". The associated time-stamp is "tsc". + * + * Return : + * 0 on success. + * -EAGAIN if channel is disabled. + * -ENOSPC if event size is too large for packet. + * -ENOBUFS if there is currently not enough space in buffer for the event. + * -EIO if data cannot be written into the buffer for any other reason. + */ + +static inline +int lib_ring_buffer_reserve(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer_ctx *ctx, + void *client_ctx) +{ + struct channel *chan = ctx->chan; + struct lib_ring_buffer *buf; + unsigned long o_begin, o_end, o_old; + size_t before_hdr_pad = 0; + + if (unlikely(atomic_read(&chan->record_disabled))) + return -EAGAIN; + + if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) + buf = per_cpu_ptr(chan->backend.buf, ctx->cpu); + else + buf = chan->backend.buf; + if (unlikely(atomic_read(&buf->record_disabled))) + return -EAGAIN; + ctx->buf = buf; + + /* + * Perform retryable operations. + */ + if (unlikely(lib_ring_buffer_try_reserve(config, ctx, client_ctx, &o_begin, + &o_end, &o_old, &before_hdr_pad))) + goto slow_path; + + if (unlikely(v_cmpxchg(config, &ctx->buf->offset, o_old, o_end) + != o_old)) + goto slow_path; + + /* + * Atomically update last_tsc. This update races against concurrent + * atomic updates, but the race will always cause supplementary full TSC + * record headers, never the opposite (missing a full TSC record header + * when it would be needed). + */ + save_last_tsc(config, ctx->buf, ctx->tsc); + + /* + * Push the reader if necessary + */ + lib_ring_buffer_reserve_push_reader(ctx->buf, chan, o_end - 1); + + /* + * Clear noref flag for this subbuffer. + */ + lib_ring_buffer_clear_noref(config, &ctx->buf->backend, + subbuf_index(o_end - 1, chan)); + + ctx->pre_offset = o_begin; + ctx->buf_offset = o_begin + before_hdr_pad; + return 0; +slow_path: + return lib_ring_buffer_reserve_slow(ctx, client_ctx); +} + +/** + * lib_ring_buffer_switch - Perform a sub-buffer switch for a per-cpu buffer. + * @config: ring buffer instance configuration. + * @buf: buffer + * @mode: buffer switch mode (SWITCH_ACTIVE or SWITCH_FLUSH) + * + * This operation is completely reentrant : can be called while tracing is + * active with absolutely no lock held. + * + * Note, however, that as a v_cmpxchg is used for some atomic operations and + * requires to be executed locally for per-CPU buffers, this function must be + * called from the CPU which owns the buffer for a ACTIVE flush, with preemption + * disabled, for RING_BUFFER_SYNC_PER_CPU configuration. + */ +static inline +void lib_ring_buffer_switch(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, enum switch_mode mode) +{ + lib_ring_buffer_switch_slow(buf, mode); +} + +/* See ring_buffer_frontend_api.h for lib_ring_buffer_reserve(). */ + +/** + * lib_ring_buffer_commit - Commit an record. + * @config: ring buffer instance configuration. + * @ctx: ring buffer context. (input arguments only) + * + * Atomic unordered slot commit. Increments the commit count in the + * specified sub-buffer, and delivers it if necessary. + */ +static inline +void lib_ring_buffer_commit(const struct lib_ring_buffer_config *config, + const struct lib_ring_buffer_ctx *ctx) +{ + struct channel *chan = ctx->chan; + struct lib_ring_buffer *buf = ctx->buf; + unsigned long offset_end = ctx->buf_offset; + unsigned long endidx = subbuf_index(offset_end - 1, chan); + unsigned long commit_count; + struct commit_counters_hot *cc_hot = &buf->commit_hot[endidx]; + + /* + * Must count record before incrementing the commit count. + */ + subbuffer_count_record(config, &buf->backend, endidx); + + /* + * Order all writes to buffer before the commit count update that will + * determine that the subbuffer is full. + */ + if (config->ipi == RING_BUFFER_IPI_BARRIER) { + /* + * Must write slot data before incrementing commit count. This + * compiler barrier is upgraded into a smp_mb() by the IPI sent + * by get_subbuf(). + */ + barrier(); + } else + smp_wmb(); + + v_add(config, ctx->slot_size, &cc_hot->cc); + + /* + * commit count read can race with concurrent OOO commit count updates. + * This is only needed for lib_ring_buffer_check_deliver (for + * non-polling delivery only) and for + * lib_ring_buffer_write_commit_counter. The race can only cause the + * counter to be read with the same value more than once, which could + * cause : + * - Multiple delivery for the same sub-buffer (which is handled + * gracefully by the reader code) if the value is for a full + * sub-buffer. It's important that we can never miss a sub-buffer + * delivery. Re-reading the value after the v_add ensures this. + * - Reading a commit_count with a higher value that what was actually + * added to it for the lib_ring_buffer_write_commit_counter call + * (again caused by a concurrent committer). It does not matter, + * because this function is interested in the fact that the commit + * count reaches back the reserve offset for a specific sub-buffer, + * which is completely independent of the order. + */ + commit_count = v_read(config, &cc_hot->cc); + + lib_ring_buffer_check_deliver(config, buf, chan, offset_end - 1, + commit_count, endidx, ctx->tsc); + /* + * Update used size at each commit. It's needed only for extracting + * ring_buffer buffers from vmcore, after crash. + */ + lib_ring_buffer_write_commit_counter(config, buf, chan, + offset_end, commit_count, cc_hot); +} + +/** + * lib_ring_buffer_try_discard_reserve - Try discarding a record. + * @config: ring buffer instance configuration. + * @ctx: ring buffer context. (input arguments only) + * + * Only succeeds if no other record has been written after the record to + * discard. If discard fails, the record must be committed to the buffer. + * + * Returns 0 upon success, -EPERM if the record cannot be discarded. + */ +static inline +int lib_ring_buffer_try_discard_reserve(const struct lib_ring_buffer_config *config, + const struct lib_ring_buffer_ctx *ctx) +{ + struct lib_ring_buffer *buf = ctx->buf; + unsigned long end_offset = ctx->pre_offset + ctx->slot_size; + + /* + * We need to ensure that if the cmpxchg succeeds and discards the + * record, the next record will record a full TSC, because it cannot + * rely on the last_tsc associated with the discarded record to detect + * overflows. The only way to ensure this is to set the last_tsc to 0 + * (assuming no 64-bit TSC overflow), which forces to write a 64-bit + * timestamp in the next record. + * + * Note: if discard fails, we must leave the TSC in the record header. + * It is needed to keep track of TSC overflows for the following + * records. + */ + save_last_tsc(config, buf, 0ULL); + + if (likely(v_cmpxchg(config, &buf->offset, end_offset, ctx->pre_offset) + != end_offset)) + return -EPERM; + else + return 0; +} + +static inline +void channel_record_disable(const struct lib_ring_buffer_config *config, + struct channel *chan) +{ + atomic_inc(&chan->record_disabled); +} + +static inline +void channel_record_enable(const struct lib_ring_buffer_config *config, + struct channel *chan) +{ + atomic_dec(&chan->record_disabled); +} + +static inline +void lib_ring_buffer_record_disable(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + atomic_inc(&buf->record_disabled); +} + +static inline +void lib_ring_buffer_record_enable(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf) +{ + atomic_dec(&buf->record_disabled); +} + +#endif /* _LIB_RING_BUFFER_FRONTEND_API_H */ diff --git a/include/ringbuffer/frontend_internal.h b/include/ringbuffer/frontend_internal.h index c14757aa..7c7c0014 100644 --- a/include/ringbuffer/frontend_internal.h +++ b/include/ringbuffer/frontend_internal.h @@ -1,2 +1,334 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * linux/ringbuffer/frontend_internal.h + * + * Ring Buffer Library Synchronization Header (internal helpers). + * + * Copyright (C) 2005-2012 Mathieu Desnoyers + * + * See ring_buffer_frontend.c for more information on wait-free algorithms. + */ + +#ifndef _LIB_RING_BUFFER_FRONTEND_INTERNAL_H +#define _LIB_RING_BUFFER_FRONTEND_INTERNAL_H + +#include +#include +#include +#include /* For per-CPU read-side iterator */ + +/* Buffer offset macros */ + +/* buf_trunc mask selects only the buffer number. */ +static inline +unsigned long buf_trunc(unsigned long offset, struct channel *chan) +{ + return offset & ~(chan->backend.buf_size - 1); + +} + +/* Select the buffer number value (counter). */ +static inline +unsigned long buf_trunc_val(unsigned long offset, struct channel *chan) +{ + return buf_trunc(offset, chan) >> chan->backend.buf_size_order; +} + +/* buf_offset mask selects only the offset within the current buffer. */ +static inline +unsigned long buf_offset(unsigned long offset, struct channel *chan) +{ + return offset & (chan->backend.buf_size - 1); +} + +/* subbuf_offset mask selects the offset within the current subbuffer. */ +static inline +unsigned long subbuf_offset(unsigned long offset, struct channel *chan) +{ + return offset & (chan->backend.subbuf_size - 1); +} + +/* subbuf_trunc mask selects the subbuffer number. */ +static inline +unsigned long subbuf_trunc(unsigned long offset, struct channel *chan) +{ + return offset & ~(chan->backend.subbuf_size - 1); +} + +/* subbuf_align aligns the offset to the next subbuffer. */ +static inline +unsigned long subbuf_align(unsigned long offset, struct channel *chan) +{ + return (offset + chan->backend.subbuf_size) + & ~(chan->backend.subbuf_size - 1); +} + +/* subbuf_index returns the index of the current subbuffer within the buffer. */ +static inline +unsigned long subbuf_index(unsigned long offset, struct channel *chan) +{ + return buf_offset(offset, chan) >> chan->backend.subbuf_size_order; +} + +/* + * Last TSC comparison functions. Check if the current TSC overflows tsc_bits + * bits from the last TSC read. When overflows are detected, the full 64-bit + * timestamp counter should be written in the record header. Reads and writes + * last_tsc atomically. + */ + +#if (BITS_PER_LONG == 32) +static inline +void save_last_tsc(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, u64 tsc) +{ + if (config->tsc_bits == 0 || config->tsc_bits == 64) + return; + + /* + * Ensure the compiler performs this update in a single instruction. + */ + v_set(config, &buf->last_tsc, (unsigned long)(tsc >> config->tsc_bits)); +} + +static inline +int last_tsc_overflow(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, u64 tsc) +{ + unsigned long tsc_shifted; + + if (config->tsc_bits == 0 || config->tsc_bits == 64) + return 0; + + tsc_shifted = (unsigned long)(tsc >> config->tsc_bits); + if (unlikely(tsc_shifted + - (unsigned long)v_read(config, &buf->last_tsc))) + return 1; + else + return 0; +} +#else +static inline +void save_last_tsc(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, u64 tsc) +{ + if (config->tsc_bits == 0 || config->tsc_bits == 64) + return; + + v_set(config, &buf->last_tsc, (unsigned long)tsc); +} + +static inline +int last_tsc_overflow(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, u64 tsc) +{ + if (config->tsc_bits == 0 || config->tsc_bits == 64) + return 0; + + if (unlikely((tsc - v_read(config, &buf->last_tsc)) + >> config->tsc_bits)) + return 1; + else + return 0; +} +#endif + +extern +int lib_ring_buffer_reserve_slow(struct lib_ring_buffer_ctx *ctx, + void *client_ctx); + +extern +void lib_ring_buffer_switch_slow(struct lib_ring_buffer *buf, + enum switch_mode mode); + +extern +void lib_ring_buffer_check_deliver_slow(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, + struct channel *chan, + unsigned long offset, + unsigned long commit_count, + unsigned long idx, + u64 tsc); + +extern +void lib_ring_buffer_switch_remote(struct lib_ring_buffer *buf); +extern +void lib_ring_buffer_switch_remote_empty(struct lib_ring_buffer *buf); +extern +void lib_ring_buffer_clear(struct lib_ring_buffer *buf); + +/* Buffer write helpers */ + +static inline +void lib_ring_buffer_reserve_push_reader(struct lib_ring_buffer *buf, + struct channel *chan, + unsigned long offset) +{ + unsigned long consumed_old, consumed_new; + + do { + consumed_old = atomic_long_read(&buf->consumed); + /* + * If buffer is in overwrite mode, push the reader consumed + * count if the write position has reached it and we are not + * at the first iteration (don't push the reader farther than + * the writer). This operation can be done concurrently by many + * writers in the same buffer, the writer being at the farthest + * write position sub-buffer index in the buffer being the one + * which will win this loop. + */ + if (unlikely(subbuf_trunc(offset, chan) + - subbuf_trunc(consumed_old, chan) + >= chan->backend.buf_size)) + consumed_new = subbuf_align(consumed_old, chan); + else + return; + } while (unlikely(atomic_long_cmpxchg(&buf->consumed, consumed_old, + consumed_new) != consumed_old)); +} + +/* + * Move consumed position to the beginning of subbuffer in which the + * write offset is. Should only be used on ring buffers that are not + * actively being written into, because clear_reader does not take into + * account the commit counters when moving the consumed position, which + * can make concurrent trace producers or consumers observe consumed + * position further than the write offset, which breaks ring buffer + * algorithm guarantees. + */ +static inline +void lib_ring_buffer_clear_reader(struct lib_ring_buffer *buf, + struct channel *chan) +{ + const struct lib_ring_buffer_config *config = &chan->backend.config; + unsigned long offset, consumed_old, consumed_new; + + do { + offset = v_read(config, &buf->offset); + consumed_old = atomic_long_read(&buf->consumed); + CHAN_WARN_ON(chan, (long) (subbuf_trunc(offset, chan) + - subbuf_trunc(consumed_old, chan)) + < 0); + consumed_new = subbuf_trunc(offset, chan); + } while (unlikely(atomic_long_cmpxchg(&buf->consumed, consumed_old, + consumed_new) != consumed_old)); +} + +static inline +int lib_ring_buffer_pending_data(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, + struct channel *chan) +{ + return !!subbuf_offset(v_read(config, &buf->offset), chan); +} + +static inline +unsigned long lib_ring_buffer_get_data_size(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, + unsigned long idx) +{ + return subbuffer_get_data_size(config, &buf->backend, idx); +} + +/* + * Check if all space reservation in a buffer have been committed. This helps + * knowing if an execution context is nested (for per-cpu buffers only). + * This is a very specific ftrace use-case, so we keep this as "internal" API. + */ +static inline +int lib_ring_buffer_reserve_committed(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, + struct channel *chan) +{ + unsigned long offset, idx, commit_count; + + CHAN_WARN_ON(chan, config->alloc != RING_BUFFER_ALLOC_PER_CPU); + CHAN_WARN_ON(chan, config->sync != RING_BUFFER_SYNC_PER_CPU); + + /* + * Read offset and commit count in a loop so they are both read + * atomically wrt interrupts. By deal with interrupt concurrency by + * restarting both reads if the offset has been pushed. Note that given + * we only have to deal with interrupt concurrency here, an interrupt + * modifying the commit count will also modify "offset", so it is safe + * to only check for offset modifications. + */ + do { + offset = v_read(config, &buf->offset); + idx = subbuf_index(offset, chan); + commit_count = v_read(config, &buf->commit_hot[idx].cc); + } while (offset != v_read(config, &buf->offset)); + + return ((buf_trunc(offset, chan) >> chan->backend.num_subbuf_order) + - (commit_count & chan->commit_count_mask) == 0); +} + +/* + * Receive end of subbuffer TSC as parameter. It has been read in the + * space reservation loop of either reserve or switch, which ensures it + * progresses monotonically with event records in the buffer. Therefore, + * it ensures that the end timestamp of a subbuffer is <= begin + * timestamp of the following subbuffers. + */ +static inline +void lib_ring_buffer_check_deliver(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, + struct channel *chan, + unsigned long offset, + unsigned long commit_count, + unsigned long idx, + u64 tsc) +{ + unsigned long old_commit_count = commit_count + - chan->backend.subbuf_size; + + /* Check if all commits have been done */ + if (unlikely((buf_trunc(offset, chan) >> chan->backend.num_subbuf_order) + - (old_commit_count & chan->commit_count_mask) == 0)) + lib_ring_buffer_check_deliver_slow(config, buf, chan, offset, + commit_count, idx, tsc); +} + +/* + * lib_ring_buffer_write_commit_counter + * + * For flight recording. must be called after commit. + * This function increments the subbuffer's commit_seq counter each time the + * commit count reaches back the reserve offset (modulo subbuffer size). It is + * useful for crash dump. + */ +static inline +void lib_ring_buffer_write_commit_counter(const struct lib_ring_buffer_config *config, + struct lib_ring_buffer *buf, + struct channel *chan, + unsigned long buf_offset, + unsigned long commit_count, + struct commit_counters_hot *cc_hot) +{ + unsigned long commit_seq_old; + + if (config->oops != RING_BUFFER_OOPS_CONSISTENCY) + return; + + /* + * subbuf_offset includes commit_count_mask. We can simply + * compare the offsets within the subbuffer without caring about + * buffer full/empty mismatch because offset is never zero here + * (subbuffer header and record headers have non-zero length). + */ + if (unlikely(subbuf_offset(buf_offset - commit_count, chan))) + return; + + commit_seq_old = v_read(config, &cc_hot->seq); + if (likely((long) (commit_seq_old - commit_count) < 0)) + v_set(config, &cc_hot->seq, commit_count); +} + +extern int lib_ring_buffer_create(struct lib_ring_buffer *buf, + struct channel_backend *chanb, int cpu); +extern void lib_ring_buffer_free(struct lib_ring_buffer *buf); + +/* Keep track of trap nesting inside ring buffer code */ +DECLARE_PER_CPU(unsigned int, lib_ring_buffer_nesting); + +#endif /* _LIB_RING_BUFFER_FRONTEND_INTERNAL_H */ diff --git a/include/ringbuffer/frontend_types.h b/include/ringbuffer/frontend_types.h index 8f6b72f2..7c209169 100644 --- a/include/ringbuffer/frontend_types.h +++ b/include/ringbuffer/frontend_types.h @@ -1,2 +1,188 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/frontend_types.h + * + * Ring Buffer Library Synchronization Header (types). + * + * Copyright (C) 2010-2012 Mathieu Desnoyers + * + * See ring_buffer_frontend.c for more information on wait-free algorithms. + */ + +#ifndef _LIB_RING_BUFFER_FRONTEND_TYPES_H +#define _LIB_RING_BUFFER_FRONTEND_TYPES_H + +#include +#include +#include +#include /* For per-CPU read-side iterator */ +#include + +/* + * A switch is done during tracing or as a final flush after tracing (so it + * won't write in the new sub-buffer). + */ +enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH }; + +/* channel-level read-side iterator */ +struct channel_iter { + /* Prio heap of buffers. Lowest timestamps at the top. */ + struct lttng_ptr_heap heap; /* Heap of struct lib_ring_buffer ptrs */ + struct list_head empty_head; /* Empty buffers linked-list head */ + int read_open; /* Opened for reading ? */ + u64 last_qs; /* Last quiescent state timestamp */ + u64 last_timestamp; /* Last timestamp (for WARN_ON) */ + int last_cpu; /* Last timestamp cpu */ + /* + * read() file operation state. + */ + unsigned long len_left; +}; + +/* channel: collection of per-cpu ring buffers. */ +struct channel { + atomic_t record_disabled; + unsigned long commit_count_mask; /* + * Commit count mask, removing + * the MSBs corresponding to + * bits used to represent the + * subbuffer index. + */ + + struct channel_backend backend; /* Associated backend */ + + unsigned long switch_timer_interval; /* Buffer flush (jiffies) */ + unsigned long read_timer_interval; /* Reader wakeup (jiffies) */ + struct lttng_cpuhp_node cpuhp_prepare; + struct lttng_cpuhp_node cpuhp_online; + struct lttng_cpuhp_node cpuhp_iter_online; + struct notifier_block tick_nohz_notifier; /* CPU nohz notifier */ + wait_queue_head_t read_wait; /* reader wait queue */ + wait_queue_head_t hp_wait; /* CPU hotplug wait queue */ + int finalized; /* Has channel been finalized */ + struct channel_iter iter; /* Channel read-side iterator */ + struct kref ref; /* Reference count */ +}; + +/* Per-subbuffer commit counters used on the hot path */ +struct commit_counters_hot { + union v_atomic cc; /* Commit counter */ + union v_atomic seq; /* Consecutive commits */ +}; + +/* Per-subbuffer commit counters used only on cold paths */ +struct commit_counters_cold { + union v_atomic cc_sb; /* Incremented _once_ at sb switch */ +}; + +/* Per-buffer read iterator */ +struct lib_ring_buffer_iter { + u64 timestamp; /* Current record timestamp */ + size_t header_len; /* Current record header length */ + size_t payload_len; /* Current record payload length */ + + struct list_head empty_node; /* Linked list of empty buffers */ + unsigned long consumed, read_offset, data_size; + enum { + ITER_GET_SUBBUF = 0, + ITER_TEST_RECORD, + ITER_NEXT_RECORD, + ITER_PUT_SUBBUF, + } state; + unsigned int allocated:1; + unsigned int read_open:1; /* Opened for reading ? */ +}; + +/* ring buffer state */ +struct lib_ring_buffer { + /* First 32 bytes cache-hot cacheline */ + union v_atomic offset; /* Current offset in the buffer */ + struct commit_counters_hot *commit_hot; + /* Commit count per sub-buffer */ + atomic_long_t consumed; /* + * Current offset in the buffer + * standard atomic access (shared) + */ + atomic_t record_disabled; + /* End of first 32 bytes cacheline */ + union v_atomic last_tsc; /* + * Last timestamp written in the buffer. + */ + + struct lib_ring_buffer_backend backend; /* Associated backend */ + + struct commit_counters_cold *commit_cold; + /* Commit count per sub-buffer */ + u64 *ts_end; /* + * timestamp_end per sub-buffer. + * Time is sampled by the + * switch_*_end() callbacks which + * are the last space reservation + * performed in the sub-buffer + * before it can be fully + * committed and delivered. This + * time value is then read by + * the deliver callback, + * performed by the last commit + * before the buffer becomes + * readable. + */ + atomic_long_t active_readers; /* + * Active readers count + * standard atomic access (shared) + */ + /* Dropped records */ + union v_atomic records_lost_full; /* Buffer full */ + union v_atomic records_lost_wrap; /* Nested wrap-around */ + union v_atomic records_lost_big; /* Events too big */ + union v_atomic records_count; /* Number of records written */ + union v_atomic records_overrun; /* Number of overwritten records */ + wait_queue_head_t read_wait; /* reader buffer-level wait queue */ + wait_queue_head_t write_wait; /* writer buffer-level wait queue (for metadata only) */ + int finalized; /* buffer has been finalized */ + struct timer_list switch_timer; /* timer for periodical switch */ + struct timer_list read_timer; /* timer for read poll */ + raw_spinlock_t raw_tick_nohz_spinlock; /* nohz entry lock/trylock */ + struct lib_ring_buffer_iter iter; /* read-side iterator */ + unsigned long get_subbuf_consumed; /* Read-side consumed */ + unsigned long prod_snapshot; /* Producer count snapshot */ + unsigned long cons_snapshot; /* Consumer count snapshot */ + unsigned int get_subbuf:1, /* Sub-buffer being held by reader */ + switch_timer_enabled:1, /* Protected by ring_buffer_nohz_lock */ + read_timer_enabled:1, /* Protected by ring_buffer_nohz_lock */ + quiescent:1; +}; + +static inline +void *channel_get_private(struct channel *chan) +{ + return chan->backend.priv; +} + +void lib_ring_buffer_lost_event_too_big(struct channel *chan); + +/* + * Issue warnings and disable channels upon internal error. + * Can receive struct lib_ring_buffer or struct lib_ring_buffer_backend + * parameters. + */ +#define CHAN_WARN_ON(c, cond) \ + ({ \ + struct channel *__chan; \ + int _____ret = unlikely(cond); \ + if (_____ret) { \ + if (__same_type(*(c), struct channel_backend)) \ + __chan = container_of((void *) (c), \ + struct channel, \ + backend); \ + else if (__same_type(*(c), struct channel)) \ + __chan = (void *) (c); \ + else \ + BUG_ON(1); \ + atomic_inc(&__chan->record_disabled); \ + WARN_ON(1); \ + } \ + _____ret; \ + }) + +#endif /* _LIB_RING_BUFFER_FRONTEND_TYPES_H */ diff --git a/include/ringbuffer/iterator.h b/include/ringbuffer/iterator.h index 08586d33..6640e695 100644 --- a/include/ringbuffer/iterator.h +++ b/include/ringbuffer/iterator.h @@ -1,2 +1,67 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/iterator.h + * + * Ring buffer and channel iterators. + * + * Copyright (C) 2010-2012 Mathieu Desnoyers + */ + +#ifndef _LIB_RING_BUFFER_ITERATOR_H +#define _LIB_RING_BUFFER_ITERATOR_H + +#include +#include +#include + +/* + * lib_ring_buffer_get_next_record advances the buffer read position to the next + * record. It returns either the size of the next record, -EAGAIN if there is + * currently no data available, or -ENODATA if no data is available and buffer + * is finalized. + */ +extern ssize_t lib_ring_buffer_get_next_record(struct channel *chan, + struct lib_ring_buffer *buf); + +/* + * channel_get_next_record advances the buffer read position to the next record. + * It returns either the size of the next record, -EAGAIN if there is currently + * no data available, or -ENODATA if no data is available and buffer is + * finalized. + * Returns the current buffer in ret_buf. + */ +extern ssize_t channel_get_next_record(struct channel *chan, + struct lib_ring_buffer **ret_buf); + +/** + * read_current_record - copy the buffer current record into dest. + * @buf: ring buffer + * @dest: destination where the record should be copied + * + * dest should be large enough to contain the record. Returns the number of + * bytes copied. + */ +static inline size_t read_current_record(struct lib_ring_buffer *buf, void *dest) +{ + return lib_ring_buffer_read(&buf->backend, buf->iter.read_offset, + dest, buf->iter.payload_len); +} + +extern int lib_ring_buffer_iterator_open(struct lib_ring_buffer *buf); +extern void lib_ring_buffer_iterator_release(struct lib_ring_buffer *buf); +extern int channel_iterator_open(struct channel *chan); +extern void channel_iterator_release(struct channel *chan); + +extern const struct file_operations channel_payload_file_operations; +extern const struct file_operations lib_ring_buffer_payload_file_operations; + +/* + * Used internally. + */ +int channel_iterator_init(struct channel *chan); +void channel_iterator_unregister_notifiers(struct channel *chan); +void channel_iterator_free(struct channel *chan); +void channel_iterator_reset(struct channel *chan); +void lib_ring_buffer_iterator_reset(struct lib_ring_buffer *buf); + +#endif /* _LIB_RING_BUFFER_ITERATOR_H */ diff --git a/include/ringbuffer/nohz.h b/include/ringbuffer/nohz.h index 33568ed5..1d28d279 100644 --- a/include/ringbuffer/nohz.h +++ b/include/ringbuffer/nohz.h @@ -1,2 +1,29 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/nohz.h + * + * Copyright (C) 2011-2012 Mathieu Desnoyers + */ + +#ifndef _LIB_RING_BUFFER_NOHZ_H +#define _LIB_RING_BUFFER_NOHZ_H + +#ifdef CONFIG_LIB_RING_BUFFER +void lib_ring_buffer_tick_nohz_flush(void); +void lib_ring_buffer_tick_nohz_stop(void); +void lib_ring_buffer_tick_nohz_restart(void); +#else +static inline void lib_ring_buffer_tick_nohz_flush(void) +{ +} + +static inline void lib_ring_buffer_tick_nohz_stop(void) +{ +} + +static inline void lib_ring_buffer_tick_nohz_restart(void) +{ +} +#endif + +#endif /* _LIB_RING_BUFFER_NOHZ_H */ diff --git a/include/ringbuffer/vatomic.h b/include/ringbuffer/vatomic.h index a5e8c76a..6fdc4d1b 100644 --- a/include/ringbuffer/vatomic.h +++ b/include/ringbuffer/vatomic.h @@ -1,2 +1,84 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/vatomic.h + * + * Copyright (C) 2010-2012 Mathieu Desnoyers + */ + +#ifndef _LIB_RING_BUFFER_VATOMIC_H +#define _LIB_RING_BUFFER_VATOMIC_H + +#include +#include + +/* + * Same data type (long) accessed differently depending on configuration. + * v field is for non-atomic access (protected by mutual exclusion). + * In the fast-path, the ring_buffer_config structure is constant, so the + * compiler can statically select the appropriate branch. + * local_t is used for per-cpu and per-thread buffers. + * atomic_long_t is used for globally shared buffers. + */ +union v_atomic { + local_t l; + atomic_long_t a; + long v; +}; + +static inline +long v_read(const struct lib_ring_buffer_config *config, union v_atomic *v_a) +{ + if (config->sync == RING_BUFFER_SYNC_PER_CPU) + return local_read(&v_a->l); + else + return atomic_long_read(&v_a->a); +} + +static inline +void v_set(const struct lib_ring_buffer_config *config, union v_atomic *v_a, + long v) +{ + if (config->sync == RING_BUFFER_SYNC_PER_CPU) + local_set(&v_a->l, v); + else + atomic_long_set(&v_a->a, v); +} + +static inline +void v_add(const struct lib_ring_buffer_config *config, long v, union v_atomic *v_a) +{ + if (config->sync == RING_BUFFER_SYNC_PER_CPU) + local_add(v, &v_a->l); + else + atomic_long_add(v, &v_a->a); +} + +static inline +void v_inc(const struct lib_ring_buffer_config *config, union v_atomic *v_a) +{ + if (config->sync == RING_BUFFER_SYNC_PER_CPU) + local_inc(&v_a->l); + else + atomic_long_inc(&v_a->a); +} + +/* + * Non-atomic decrement. Only used by reader, apply to reader-owned subbuffer. + */ +static inline +void _v_dec(const struct lib_ring_buffer_config *config, union v_atomic *v_a) +{ + --v_a->v; +} + +static inline +long v_cmpxchg(const struct lib_ring_buffer_config *config, union v_atomic *v_a, + long old, long _new) +{ + if (config->sync == RING_BUFFER_SYNC_PER_CPU) + return local_cmpxchg(&v_a->l, old, _new); + else + return atomic_long_cmpxchg(&v_a->a, old, _new); +} + +#endif /* _LIB_RING_BUFFER_VATOMIC_H */ diff --git a/include/ringbuffer/vfs.h b/include/ringbuffer/vfs.h index 65918408..ee23a624 100644 --- a/include/ringbuffer/vfs.h +++ b/include/ringbuffer/vfs.h @@ -1,2 +1,166 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) */ -#include +/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) + * + * lib/ringbuffer/vfs.h + * + * Wait-free ring buffer VFS file operations. + * + * Copyright (C) 2005-2012 Mathieu Desnoyers + */ + +#ifndef _LIB_RING_BUFFER_VFS_H +#define _LIB_RING_BUFFER_VFS_H + +#include +#include + +/* VFS API */ + +extern const struct file_operations lib_ring_buffer_file_operations; + +/* + * Internal file operations. + */ + +struct lib_ring_buffer; + +int lib_ring_buffer_open(struct inode *inode, struct file *file, + struct lib_ring_buffer *buf); +int lib_ring_buffer_release(struct inode *inode, struct file *file, + struct lib_ring_buffer *buf); +unsigned int lib_ring_buffer_poll(struct file *filp, poll_table *wait, + struct lib_ring_buffer *buf); +ssize_t lib_ring_buffer_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags, struct lib_ring_buffer *buf); +int lib_ring_buffer_mmap(struct file *filp, struct vm_area_struct *vma, + struct lib_ring_buffer *buf); + +/* Ring Buffer ioctl() and ioctl numbers */ +long lib_ring_buffer_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg, struct lib_ring_buffer *buf); +#ifdef CONFIG_COMPAT +long lib_ring_buffer_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg, struct lib_ring_buffer *buf); +#endif + +ssize_t vfs_lib_ring_buffer_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, unsigned int flags); +loff_t vfs_lib_ring_buffer_no_llseek(struct file *file, loff_t offset, + int origin); +int vfs_lib_ring_buffer_mmap(struct file *filp, struct vm_area_struct *vma); +ssize_t vfs_lib_ring_buffer_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags); + +/* + * Use RING_BUFFER_GET_NEXT_SUBBUF / RING_BUFFER_PUT_NEXT_SUBBUF to read and + * consume sub-buffers sequentially. + * + * Reading sub-buffers without consuming them can be performed with: + * + * RING_BUFFER_SNAPSHOT + * RING_BUFFER_SNAPSHOT_GET_CONSUMED + * RING_BUFFER_SNAPSHOT_GET_PRODUCED + * + * to get the offset range to consume, and then by passing each sub-buffer + * offset to RING_BUFFER_GET_SUBBUF, read the sub-buffer, and then release it + * with RING_BUFFER_PUT_SUBBUF. + * + * Note that the "snapshot" API can be used to read the sub-buffer in reverse + * order, which is useful for flight recorder snapshots. + */ + +/* Get a snapshot of the current ring buffer producer and consumer positions */ +#define RING_BUFFER_SNAPSHOT _IO(0xF6, 0x00) +/* Get the consumer position (iteration start) */ +#define RING_BUFFER_SNAPSHOT_GET_CONSUMED _IOR(0xF6, 0x01, unsigned long) +/* Get the producer position (iteration end) */ +#define RING_BUFFER_SNAPSHOT_GET_PRODUCED _IOR(0xF6, 0x02, unsigned long) +/* Get exclusive read access to the specified sub-buffer position */ +#define RING_BUFFER_GET_SUBBUF _IOW(0xF6, 0x03, unsigned long) +/* Release exclusive sub-buffer access */ +#define RING_BUFFER_PUT_SUBBUF _IO(0xF6, 0x04) + +/* Get exclusive read access to the next sub-buffer that can be read. */ +#define RING_BUFFER_GET_NEXT_SUBBUF _IO(0xF6, 0x05) +/* Release exclusive sub-buffer access, move consumer forward. */ +#define RING_BUFFER_PUT_NEXT_SUBBUF _IO(0xF6, 0x06) +/* returns the size of the current sub-buffer, without padding (for mmap). */ +#define RING_BUFFER_GET_SUBBUF_SIZE _IOR(0xF6, 0x07, unsigned long) +/* returns the size of the current sub-buffer, with padding (for splice). */ +#define RING_BUFFER_GET_PADDED_SUBBUF_SIZE _IOR(0xF6, 0x08, unsigned long) +/* returns the maximum size for sub-buffers. */ +#define RING_BUFFER_GET_MAX_SUBBUF_SIZE _IOR(0xF6, 0x09, unsigned long) +/* returns the length to mmap. */ +#define RING_BUFFER_GET_MMAP_LEN _IOR(0xF6, 0x0A, unsigned long) +/* returns the offset of the subbuffer belonging to the mmap reader. */ +#define RING_BUFFER_GET_MMAP_READ_OFFSET _IOR(0xF6, 0x0B, unsigned long) +/* Flush the current sub-buffer, if non-empty. */ +#define RING_BUFFER_FLUSH _IO(0xF6, 0x0C) +/* Get the current version of the metadata cache (after a get_next). */ +#define RING_BUFFER_GET_METADATA_VERSION _IOR(0xF6, 0x0D, uint64_t) +/* + * Get a snapshot of the current ring buffer producer and consumer positions, + * regardless of whether or not the two positions are contained within the same + * sub-buffer. + */ +#define RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS _IO(0xF6, 0x0E) +/* Flush the current sub-buffer, even if empty. */ +#define RING_BUFFER_FLUSH_EMPTY _IO(0xF6, 0x0F) +/* + * Reset the position of what has been consumed from the metadata cache to 0 + * so it can be read again. + */ +#define RING_BUFFER_METADATA_CACHE_DUMP _IO(0xF6, 0x10) +/* Clear ring buffer content. */ +#define RING_BUFFER_CLEAR _IO(0xF6, 0x11) + +#ifdef CONFIG_COMPAT +/* Get a snapshot of the current ring buffer producer and consumer positions */ +#define RING_BUFFER_COMPAT_SNAPSHOT RING_BUFFER_SNAPSHOT +/* Get the consumer position (iteration start) */ +#define RING_BUFFER_COMPAT_SNAPSHOT_GET_CONSUMED \ + _IOR(0xF6, 0x01, compat_ulong_t) +/* Get the producer position (iteration end) */ +#define RING_BUFFER_COMPAT_SNAPSHOT_GET_PRODUCED \ + _IOR(0xF6, 0x02, compat_ulong_t) +/* Get exclusive read access to the specified sub-buffer position */ +#define RING_BUFFER_COMPAT_GET_SUBBUF _IOW(0xF6, 0x03, compat_ulong_t) +/* Release exclusive sub-buffer access */ +#define RING_BUFFER_COMPAT_PUT_SUBBUF RING_BUFFER_PUT_SUBBUF + +/* Get exclusive read access to the next sub-buffer that can be read. */ +#define RING_BUFFER_COMPAT_GET_NEXT_SUBBUF RING_BUFFER_GET_NEXT_SUBBUF +/* Release exclusive sub-buffer access, move consumer forward. */ +#define RING_BUFFER_COMPAT_PUT_NEXT_SUBBUF RING_BUFFER_PUT_NEXT_SUBBUF +/* returns the size of the current sub-buffer, without padding (for mmap). */ +#define RING_BUFFER_COMPAT_GET_SUBBUF_SIZE _IOR(0xF6, 0x07, compat_ulong_t) +/* returns the size of the current sub-buffer, with padding (for splice). */ +#define RING_BUFFER_COMPAT_GET_PADDED_SUBBUF_SIZE \ + _IOR(0xF6, 0x08, compat_ulong_t) +/* returns the maximum size for sub-buffers. */ +#define RING_BUFFER_COMPAT_GET_MAX_SUBBUF_SIZE _IOR(0xF6, 0x09, compat_ulong_t) +/* returns the length to mmap. */ +#define RING_BUFFER_COMPAT_GET_MMAP_LEN _IOR(0xF6, 0x0A, compat_ulong_t) +/* returns the offset of the subbuffer belonging to the mmap reader. */ +#define RING_BUFFER_COMPAT_GET_MMAP_READ_OFFSET _IOR(0xF6, 0x0B, compat_ulong_t) +/* Flush the current sub-buffer, if non-empty. */ +#define RING_BUFFER_COMPAT_FLUSH RING_BUFFER_FLUSH +/* Get the current version of the metadata cache (after a get_next). */ +#define RING_BUFFER_COMPAT_GET_METADATA_VERSION RING_BUFFER_GET_METADATA_VERSION +/* + * Get a snapshot of the current ring buffer producer and consumer positions, + * regardless of whether or not the two positions are contained within the same + * sub-buffer. + */ +#define RING_BUFFER_COMPAT_SNAPSHOT_SAMPLE_POSITIONS \ + RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS +/* Flush the current sub-buffer, even if empty. */ +#define RING_BUFFER_COMPAT_FLUSH_EMPTY \ + RING_BUFFER_FLUSH_EMPTY +/* Clear ring buffer content. */ +#define RING_BUFFER_COMPAT_CLEAR \ + RING_BUFFER_CLEAR +#endif /* CONFIG_COMPAT */ + +#endif /* _LIB_RING_BUFFER_VFS_H */ diff --git a/wrapper/irqflags.h b/include/wrapper/irqflags.h similarity index 100% rename from wrapper/irqflags.h rename to include/wrapper/irqflags.h diff --git a/wrapper/trace-clock.h b/include/wrapper/trace-clock.h similarity index 98% rename from wrapper/trace-clock.h rename to include/wrapper/trace-clock.h index 28ee90eb..223de5c8 100644 --- a/wrapper/trace-clock.h +++ b/include/wrapper/trace-clock.h @@ -23,8 +23,8 @@ #include #include #include -#include -#include +#include +#include #include extern struct lttng_trace_clock *lttng_trace_clock; diff --git a/wrapper/user_namespace.h b/include/wrapper/user_namespace.h similarity index 100% rename from wrapper/user_namespace.h rename to include/wrapper/user_namespace.h diff --git a/lib/Kbuild b/lib/Kbuild index 6c731136..c03ac4e3 100644 --- a/lib/Kbuild +++ b/lib/Kbuild @@ -4,7 +4,7 @@ TOP_LTTNG_MODULES_DIR := $(shell dirname $(lastword $(MAKEFILE_LIST)))/.. include $(TOP_LTTNG_MODULES_DIR)/Kbuild.common -ccflags-y += -I$(TOP_LTTNG_MODULES_DIR) +ccflags-y += -I$(TOP_LTTNG_MODULES_DIR)/include obj-$(CONFIG_LTTNG) += lttng-lib-ring-buffer.o diff --git a/lib/prio_heap/lttng_prio_heap.c b/lib/prio_heap/lttng_prio_heap.c index 4c62782f..66bee9cc 100644 --- a/lib/prio_heap/lttng_prio_heap.c +++ b/lib/prio_heap/lttng_prio_heap.c @@ -8,7 +8,7 @@ */ #include -#include +#include #include #ifdef DEBUG_HEAP diff --git a/lib/ringbuffer/api.h b/lib/ringbuffer/api.h deleted file mode 100644 index 13261031..00000000 --- a/lib/ringbuffer/api.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/api.h - * - * Ring Buffer API. - * - * Copyright (C) 2010-2012 Mathieu Desnoyers - */ - -#ifndef _LIB_RING_BUFFER_API_H -#define _LIB_RING_BUFFER_API_H - -#include -#include -#include - -/* - * ring_buffer_frontend_api.h contains static inline functions that depend on - * client static inlines. Hence the inclusion of this >api> header only - * within the client. - */ -#include - -#endif /* _LIB_RING_BUFFER_API_H */ diff --git a/lib/ringbuffer/backend.h b/lib/ringbuffer/backend.h deleted file mode 100644 index 3d2abd77..00000000 --- a/lib/ringbuffer/backend.h +++ /dev/null @@ -1,463 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/backend.h - * - * Ring buffer backend (API). - * - * Copyright (C) 2010-2012 Mathieu Desnoyers - * - * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by - * the reader in flight recorder mode. - */ - -#ifndef _LIB_RING_BUFFER_BACKEND_H -#define _LIB_RING_BUFFER_BACKEND_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Internal helpers */ -#include -#include - -/* Ring buffer backend API */ - -/* Ring buffer backend access (read/write) */ - -extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb, - size_t offset, void *dest, size_t len); - -extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb, - size_t offset, void __user *dest, - size_t len); - -extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb, - size_t offset, void *dest, size_t len); - -extern unsigned long * -lib_ring_buffer_read_get_pfn(struct lib_ring_buffer_backend *bufb, size_t offset, - void ***virt); - -/* - * Return the address where a given offset is located. - * Should be used to get the current subbuffer header pointer. Given we know - * it's never on a page boundary, it's safe to write directly to this address, - * as long as the write is never bigger than a page size. - */ -extern void * -lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb, - size_t offset); -extern void * -lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb, - size_t offset); - -/** - * lib_ring_buffer_write - write data to a buffer backend - * @config : ring buffer instance configuration - * @ctx: ring buffer context. (input arguments only) - * @src : source pointer to copy from - * @len : length of data to copy - * - * This function copies "len" bytes of data from a source pointer to a buffer - * backend, at the current context offset. This is more or less a buffer - * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write) - * if copy is crossing a page boundary. - */ -static inline __attribute__((always_inline)) -void lib_ring_buffer_write(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_ctx *ctx, - const void *src, size_t len) -{ - struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; - struct channel_backend *chanb = &ctx->chan->backend; - size_t index, pagecpy; - size_t offset = ctx->buf_offset; - struct lib_ring_buffer_backend_pages *backend_pages; - - if (unlikely(!len)) - return; - backend_pages = - lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); - offset &= chanb->buf_size - 1; - index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; - pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); - if (likely(pagecpy == len)) - lib_ring_buffer_do_copy(config, - backend_pages->p[index].virt - + (offset & ~PAGE_MASK), - src, len); - else - _lib_ring_buffer_write(bufb, offset, src, len, 0); - ctx->buf_offset += len; -} - -/** - * lib_ring_buffer_memset - write len bytes of c to a buffer backend - * @config : ring buffer instance configuration - * @bufb : ring buffer backend - * @offset : offset within the buffer - * @c : the byte to copy - * @len : number of bytes to copy - * - * This function writes "len" bytes of "c" to a buffer backend, at a specific - * offset. This is more or less a buffer backend-specific memset() operation. - * Calls the slow path (_ring_buffer_memset) if write is crossing a page - * boundary. - */ -static inline -void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_ctx *ctx, int c, size_t len) -{ - - struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; - struct channel_backend *chanb = &ctx->chan->backend; - size_t index, pagecpy; - size_t offset = ctx->buf_offset; - struct lib_ring_buffer_backend_pages *backend_pages; - - if (unlikely(!len)) - return; - backend_pages = - lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); - offset &= chanb->buf_size - 1; - index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; - pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); - if (likely(pagecpy == len)) - lib_ring_buffer_do_memset(backend_pages->p[index].virt - + (offset & ~PAGE_MASK), - c, len); - else - _lib_ring_buffer_memset(bufb, offset, c, len, 0); - ctx->buf_offset += len; -} - -/* - * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL - * terminating character is found in @src. Returns the number of bytes - * copied. Does *not* terminate @dest with NULL terminating character. - */ -static inline __attribute__((always_inline)) -size_t lib_ring_buffer_do_strcpy(const struct lib_ring_buffer_config *config, - char *dest, const char *src, size_t len) -{ - size_t count; - - for (count = 0; count < len; count++) { - char c; - - /* - * Only read source character once, in case it is - * modified concurrently. - */ - c = READ_ONCE(src[count]); - if (!c) - break; - lib_ring_buffer_do_copy(config, &dest[count], &c, 1); - } - return count; -} - -/* - * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL - * terminating character is found in @src, or when a fault occurs. - * Returns the number of bytes copied. Does *not* terminate @dest with - * NULL terminating character. - * - * This function deals with userspace pointers, it should never be called - * directly without having the src pointer checked with access_ok() - * previously. - */ -static inline __attribute__((always_inline)) -size_t lib_ring_buffer_do_strcpy_from_user_inatomic(const struct lib_ring_buffer_config *config, - char *dest, const char __user *src, size_t len) -{ - size_t count; - - for (count = 0; count < len; count++) { - int ret; - char c; - - ret = __copy_from_user_inatomic(&c, src + count, 1); - if (ret || !c) - break; - lib_ring_buffer_do_copy(config, &dest[count], &c, 1); - } - return count; -} - -/** - * lib_ring_buffer_strcpy - write string data to a buffer backend - * @config : ring buffer instance configuration - * @ctx: ring buffer context. (input arguments only) - * @src : source pointer to copy from - * @len : length of data to copy - * @pad : character to use for padding - * - * This function copies @len - 1 bytes of string data from a source - * pointer to a buffer backend, followed by a terminating '\0' - * character, at the current context offset. This is more or less a - * buffer backend-specific strncpy() operation. If a terminating '\0' - * character is found in @src before @len - 1 characters are copied, pad - * the buffer with @pad characters (e.g. '#'). Calls the slow path - * (_ring_buffer_strcpy) if copy is crossing a page boundary. - */ -static inline -void lib_ring_buffer_strcpy(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_ctx *ctx, - const char *src, size_t len, int pad) -{ - struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; - struct channel_backend *chanb = &ctx->chan->backend; - size_t index, pagecpy; - size_t offset = ctx->buf_offset; - struct lib_ring_buffer_backend_pages *backend_pages; - - if (unlikely(!len)) - return; - backend_pages = - lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); - offset &= chanb->buf_size - 1; - index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; - pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); - if (likely(pagecpy == len)) { - size_t count; - - count = lib_ring_buffer_do_strcpy(config, - backend_pages->p[index].virt - + (offset & ~PAGE_MASK), - src, len - 1); - offset += count; - /* Padding */ - if (unlikely(count < len - 1)) { - size_t pad_len = len - 1 - count; - - lib_ring_buffer_do_memset(backend_pages->p[index].virt - + (offset & ~PAGE_MASK), - pad, pad_len); - offset += pad_len; - } - /* Ending '\0' */ - lib_ring_buffer_do_memset(backend_pages->p[index].virt - + (offset & ~PAGE_MASK), - '\0', 1); - } else { - _lib_ring_buffer_strcpy(bufb, offset, src, len, 0, pad); - } - ctx->buf_offset += len; -} - -/** - * lib_ring_buffer_copy_from_user_inatomic - write userspace data to a buffer backend - * @config : ring buffer instance configuration - * @ctx: ring buffer context. (input arguments only) - * @src : userspace source pointer to copy from - * @len : length of data to copy - * - * This function copies "len" bytes of data from a userspace pointer to a - * buffer backend, at the current context offset. This is more or less a buffer - * backend-specific memcpy() operation. Calls the slow path - * (_ring_buffer_write_from_user_inatomic) if copy is crossing a page boundary. - * Disable the page fault handler to ensure we never try to take the mmap_sem. - */ -static inline __attribute__((always_inline)) -void lib_ring_buffer_copy_from_user_inatomic(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_ctx *ctx, - const void __user *src, size_t len) -{ - struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; - struct channel_backend *chanb = &ctx->chan->backend; - size_t index, pagecpy; - size_t offset = ctx->buf_offset; - struct lib_ring_buffer_backend_pages *backend_pages; - unsigned long ret; - mm_segment_t old_fs = get_fs(); - - if (unlikely(!len)) - return; - backend_pages = - lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); - offset &= chanb->buf_size - 1; - index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; - pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); - - set_fs(KERNEL_DS); - pagefault_disable(); - if (unlikely(!access_ok(src, len))) - goto fill_buffer; - - if (likely(pagecpy == len)) { - ret = lib_ring_buffer_do_copy_from_user_inatomic( - backend_pages->p[index].virt + (offset & ~PAGE_MASK), - src, len); - if (unlikely(ret > 0)) { - /* Copy failed. */ - goto fill_buffer; - } - } else { - _lib_ring_buffer_copy_from_user_inatomic(bufb, offset, src, len, 0); - } - pagefault_enable(); - set_fs(old_fs); - ctx->buf_offset += len; - - return; - -fill_buffer: - pagefault_enable(); - set_fs(old_fs); - /* - * In the error path we call the slow path version to avoid - * the pollution of static inline code. - */ - _lib_ring_buffer_memset(bufb, offset, 0, len, 0); -} - -/** - * lib_ring_buffer_strcpy_from_user_inatomic - write userspace string data to a buffer backend - * @config : ring buffer instance configuration - * @ctx: ring buffer context (input arguments only) - * @src : userspace source pointer to copy from - * @len : length of data to copy - * @pad : character to use for padding - * - * This function copies @len - 1 bytes of string data from a userspace - * source pointer to a buffer backend, followed by a terminating '\0' - * character, at the current context offset. This is more or less a - * buffer backend-specific strncpy() operation. If a terminating '\0' - * character is found in @src before @len - 1 characters are copied, pad - * the buffer with @pad characters (e.g. '#'). Calls the slow path - * (_ring_buffer_strcpy_from_user_inatomic) if copy is crossing a page - * boundary. Disable the page fault handler to ensure we never try to - * take the mmap_sem. - */ -static inline -void lib_ring_buffer_strcpy_from_user_inatomic(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_ctx *ctx, - const void __user *src, size_t len, int pad) -{ - struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; - struct channel_backend *chanb = &ctx->chan->backend; - size_t index, pagecpy; - size_t offset = ctx->buf_offset; - struct lib_ring_buffer_backend_pages *backend_pages; - mm_segment_t old_fs = get_fs(); - - if (unlikely(!len)) - return; - backend_pages = - lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); - offset &= chanb->buf_size - 1; - index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT; - pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK); - - set_fs(KERNEL_DS); - pagefault_disable(); - if (unlikely(!access_ok(src, len))) - goto fill_buffer; - - if (likely(pagecpy == len)) { - size_t count; - - count = lib_ring_buffer_do_strcpy_from_user_inatomic(config, - backend_pages->p[index].virt - + (offset & ~PAGE_MASK), - src, len - 1); - offset += count; - /* Padding */ - if (unlikely(count < len - 1)) { - size_t pad_len = len - 1 - count; - - lib_ring_buffer_do_memset(backend_pages->p[index].virt - + (offset & ~PAGE_MASK), - pad, pad_len); - offset += pad_len; - } - /* Ending '\0' */ - lib_ring_buffer_do_memset(backend_pages->p[index].virt - + (offset & ~PAGE_MASK), - '\0', 1); - } else { - _lib_ring_buffer_strcpy_from_user_inatomic(bufb, offset, src, - len, 0, pad); - } - pagefault_enable(); - set_fs(old_fs); - ctx->buf_offset += len; - - return; - -fill_buffer: - pagefault_enable(); - set_fs(old_fs); - /* - * In the error path we call the slow path version to avoid - * the pollution of static inline code. - */ - _lib_ring_buffer_memset(bufb, offset, pad, len - 1, 0); - offset += len - 1; - _lib_ring_buffer_memset(bufb, offset, '\0', 1, 0); -} - -/* - * This accessor counts the number of unread records in a buffer. - * It only provides a consistent value if no reads not writes are performed - * concurrently. - */ -static inline -unsigned long lib_ring_buffer_get_records_unread( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - struct lib_ring_buffer_backend *bufb = &buf->backend; - struct lib_ring_buffer_backend_pages *pages; - unsigned long records_unread = 0, sb_bindex, id; - unsigned int i; - - for (i = 0; i < bufb->chan->backend.num_subbuf; i++) { - id = bufb->buf_wsb[i].id; - sb_bindex = subbuffer_id_get_index(config, id); - pages = bufb->array[sb_bindex]; - records_unread += v_read(config, &pages->records_unread); - } - if (config->mode == RING_BUFFER_OVERWRITE) { - id = bufb->buf_rsb.id; - sb_bindex = subbuffer_id_get_index(config, id); - pages = bufb->array[sb_bindex]; - records_unread += v_read(config, &pages->records_unread); - } - return records_unread; -} - -/* - * We use __copy_from_user_inatomic to copy userspace data after - * checking with access_ok() and disabling page faults. - * - * Return 0 if OK, nonzero on error. - */ -static inline -unsigned long lib_ring_buffer_copy_from_user_check_nofault(void *dest, - const void __user *src, - unsigned long len) -{ - unsigned long ret; - mm_segment_t old_fs; - - if (!access_ok(src, len)) - return 1; - old_fs = get_fs(); - set_fs(KERNEL_DS); - pagefault_disable(); - ret = __copy_from_user_inatomic(dest, src, len); - pagefault_enable(); - set_fs(old_fs); - return ret; -} - -#endif /* _LIB_RING_BUFFER_BACKEND_H */ diff --git a/lib/ringbuffer/backend_internal.h b/lib/ringbuffer/backend_internal.h deleted file mode 100644 index 3a2ce26f..00000000 --- a/lib/ringbuffer/backend_internal.h +++ /dev/null @@ -1,531 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/backend_internal.h - * - * Ring buffer backend (internal helpers). - * - * Copyright (C) 2008-2012 Mathieu Desnoyers - */ - -#ifndef _LIB_RING_BUFFER_BACKEND_INTERNAL_H -#define _LIB_RING_BUFFER_BACKEND_INTERNAL_H - -#include -#include -#include -#include -#include - -/* Ring buffer backend API presented to the frontend */ - -/* Ring buffer and channel backend create/free */ - -int lib_ring_buffer_backend_create(struct lib_ring_buffer_backend *bufb, - struct channel_backend *chan, int cpu); -void channel_backend_unregister_notifiers(struct channel_backend *chanb); -void lib_ring_buffer_backend_free(struct lib_ring_buffer_backend *bufb); -int channel_backend_init(struct channel_backend *chanb, - const char *name, - const struct lib_ring_buffer_config *config, - void *priv, size_t subbuf_size, - size_t num_subbuf); -void channel_backend_free(struct channel_backend *chanb); - -void lib_ring_buffer_backend_reset(struct lib_ring_buffer_backend *bufb); -void channel_backend_reset(struct channel_backend *chanb); - -int lib_ring_buffer_backend_init(void); -void lib_ring_buffer_backend_exit(void); - -extern void _lib_ring_buffer_write(struct lib_ring_buffer_backend *bufb, - size_t offset, const void *src, size_t len, - size_t pagecpy); -extern void _lib_ring_buffer_memset(struct lib_ring_buffer_backend *bufb, - size_t offset, int c, size_t len, - size_t pagecpy); -extern void _lib_ring_buffer_strcpy(struct lib_ring_buffer_backend *bufb, - size_t offset, const char *src, size_t len, - size_t pagecpy, int pad); -extern void _lib_ring_buffer_copy_from_user_inatomic(struct lib_ring_buffer_backend *bufb, - size_t offset, const void *src, - size_t len, size_t pagecpy); -extern void _lib_ring_buffer_strcpy_from_user_inatomic(struct lib_ring_buffer_backend *bufb, - size_t offset, const char __user *src, size_t len, - size_t pagecpy, int pad); - -/* - * Subbuffer ID bits for overwrite mode. Need to fit within a single word to be - * exchanged atomically. - * - * Top half word, except lowest bit, belongs to "offset", which is used to keep - * to count the produced buffers. For overwrite mode, this provides the - * consumer with the capacity to read subbuffers in order, handling the - * situation where producers would write up to 2^15 buffers (or 2^31 for 64-bit - * systems) concurrently with a single execution of get_subbuf (between offset - * sampling and subbuffer ID exchange). - */ - -#define HALF_ULONG_BITS (BITS_PER_LONG >> 1) - -#define SB_ID_OFFSET_SHIFT (HALF_ULONG_BITS + 1) -#define SB_ID_OFFSET_COUNT (1UL << SB_ID_OFFSET_SHIFT) -#define SB_ID_OFFSET_MASK (~(SB_ID_OFFSET_COUNT - 1)) -/* - * Lowest bit of top word half belongs to noref. Used only for overwrite mode. - */ -#define SB_ID_NOREF_SHIFT (SB_ID_OFFSET_SHIFT - 1) -#define SB_ID_NOREF_COUNT (1UL << SB_ID_NOREF_SHIFT) -#define SB_ID_NOREF_MASK SB_ID_NOREF_COUNT -/* - * In overwrite mode: lowest half of word is used for index. - * Limit of 2^16 subbuffers per buffer on 32-bit, 2^32 on 64-bit. - * In producer-consumer mode: whole word used for index. - */ -#define SB_ID_INDEX_SHIFT 0 -#define SB_ID_INDEX_COUNT (1UL << SB_ID_INDEX_SHIFT) -#define SB_ID_INDEX_MASK (SB_ID_NOREF_COUNT - 1) - -/* - * Construct the subbuffer id from offset, index and noref. Use only the index - * for producer-consumer mode (offset and noref are only used in overwrite - * mode). - */ -static inline -unsigned long subbuffer_id(const struct lib_ring_buffer_config *config, - unsigned long offset, unsigned long noref, - unsigned long index) -{ - if (config->mode == RING_BUFFER_OVERWRITE) - return (offset << SB_ID_OFFSET_SHIFT) - | (noref << SB_ID_NOREF_SHIFT) - | index; - else - return index; -} - -/* - * Compare offset with the offset contained within id. Return 1 if the offset - * bits are identical, else 0. - */ -static inline -int subbuffer_id_compare_offset(const struct lib_ring_buffer_config *config, - unsigned long id, unsigned long offset) -{ - return (id & SB_ID_OFFSET_MASK) == (offset << SB_ID_OFFSET_SHIFT); -} - -static inline -unsigned long subbuffer_id_get_index(const struct lib_ring_buffer_config *config, - unsigned long id) -{ - if (config->mode == RING_BUFFER_OVERWRITE) - return id & SB_ID_INDEX_MASK; - else - return id; -} - -static inline -unsigned long subbuffer_id_is_noref(const struct lib_ring_buffer_config *config, - unsigned long id) -{ - if (config->mode == RING_BUFFER_OVERWRITE) - return !!(id & SB_ID_NOREF_MASK); - else - return 1; -} - -/* - * Only used by reader on subbuffer ID it has exclusive access to. No volatile - * needed. - */ -static inline -void subbuffer_id_set_noref(const struct lib_ring_buffer_config *config, - unsigned long *id) -{ - if (config->mode == RING_BUFFER_OVERWRITE) - *id |= SB_ID_NOREF_MASK; -} - -static inline -void subbuffer_id_set_noref_offset(const struct lib_ring_buffer_config *config, - unsigned long *id, unsigned long offset) -{ - unsigned long tmp; - - if (config->mode == RING_BUFFER_OVERWRITE) { - tmp = *id; - tmp &= ~SB_ID_OFFSET_MASK; - tmp |= offset << SB_ID_OFFSET_SHIFT; - tmp |= SB_ID_NOREF_MASK; - /* Volatile store, read concurrently by readers. */ - WRITE_ONCE(*id, tmp); - } -} - -/* No volatile access, since already used locally */ -static inline -void subbuffer_id_clear_noref(const struct lib_ring_buffer_config *config, - unsigned long *id) -{ - if (config->mode == RING_BUFFER_OVERWRITE) - *id &= ~SB_ID_NOREF_MASK; -} - -/* - * For overwrite mode, cap the number of subbuffers per buffer to: - * 2^16 on 32-bit architectures - * 2^32 on 64-bit architectures - * This is required to fit in the index part of the ID. Return 0 on success, - * -EPERM on failure. - */ -static inline -int subbuffer_id_check_index(const struct lib_ring_buffer_config *config, - unsigned long num_subbuf) -{ - if (config->mode == RING_BUFFER_OVERWRITE) - return (num_subbuf > (1UL << HALF_ULONG_BITS)) ? -EPERM : 0; - else - return 0; -} - -static inline -void lib_ring_buffer_backend_get_pages(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_ctx *ctx, - struct lib_ring_buffer_backend_pages **backend_pages) -{ - struct lib_ring_buffer_backend *bufb = &ctx->buf->backend; - struct channel_backend *chanb = &ctx->chan->backend; - size_t sbidx, offset = ctx->buf_offset; - unsigned long sb_bindex, id; - struct lib_ring_buffer_backend_pages *rpages; - - offset &= chanb->buf_size - 1; - sbidx = offset >> chanb->subbuf_size_order; - id = bufb->buf_wsb[sbidx].id; - sb_bindex = subbuffer_id_get_index(config, id); - rpages = bufb->array[sb_bindex]; - CHAN_WARN_ON(ctx->chan, - config->mode == RING_BUFFER_OVERWRITE - && subbuffer_id_is_noref(config, id)); - *backend_pages = rpages; -} - -/* Get backend pages from cache. */ -static inline -struct lib_ring_buffer_backend_pages * - lib_ring_buffer_get_backend_pages_from_ctx(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_ctx *ctx) -{ - return ctx->backend_pages; -} - -/* - * The ring buffer can count events recorded and overwritten per buffer, - * but it is disabled by default due to its performance overhead. - */ -#ifdef LTTNG_RING_BUFFER_COUNT_EVENTS -static inline -void subbuffer_count_record(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) -{ - unsigned long sb_bindex; - - sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); - v_inc(config, &bufb->array[sb_bindex]->records_commit); -} -#else /* LTTNG_RING_BUFFER_COUNT_EVENTS */ -static inline -void subbuffer_count_record(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) -{ -} -#endif /* #else LTTNG_RING_BUFFER_COUNT_EVENTS */ - -/* - * Reader has exclusive subbuffer access for record consumption. No need to - * perform the decrement atomically. - */ -static inline -void subbuffer_consume_record(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb) -{ - unsigned long sb_bindex; - - sb_bindex = subbuffer_id_get_index(config, bufb->buf_rsb.id); - CHAN_WARN_ON(bufb->chan, - !v_read(config, &bufb->array[sb_bindex]->records_unread)); - /* Non-atomic decrement protected by exclusive subbuffer access */ - _v_dec(config, &bufb->array[sb_bindex]->records_unread); - v_inc(config, &bufb->records_read); -} - -static inline -unsigned long subbuffer_get_records_count( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) -{ - unsigned long sb_bindex; - - sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); - return v_read(config, &bufb->array[sb_bindex]->records_commit); -} - -/* - * Must be executed at subbuffer delivery when the writer has _exclusive_ - * subbuffer access. See lib_ring_buffer_check_deliver() for details. - * lib_ring_buffer_get_records_count() must be called to get the records - * count before this function, because it resets the records_commit - * count. - */ -static inline -unsigned long subbuffer_count_records_overrun( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) -{ - struct lib_ring_buffer_backend_pages *pages; - unsigned long overruns, sb_bindex; - - sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); - pages = bufb->array[sb_bindex]; - overruns = v_read(config, &pages->records_unread); - v_set(config, &pages->records_unread, - v_read(config, &pages->records_commit)); - v_set(config, &pages->records_commit, 0); - - return overruns; -} - -static inline -void subbuffer_set_data_size(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx, - unsigned long data_size) -{ - struct lib_ring_buffer_backend_pages *pages; - unsigned long sb_bindex; - - sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); - pages = bufb->array[sb_bindex]; - pages->data_size = data_size; -} - -static inline -unsigned long subbuffer_get_read_data_size( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb) -{ - struct lib_ring_buffer_backend_pages *pages; - unsigned long sb_bindex; - - sb_bindex = subbuffer_id_get_index(config, bufb->buf_rsb.id); - pages = bufb->array[sb_bindex]; - return pages->data_size; -} - -static inline -unsigned long subbuffer_get_data_size( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) -{ - struct lib_ring_buffer_backend_pages *pages; - unsigned long sb_bindex; - - sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id); - pages = bufb->array[sb_bindex]; - return pages->data_size; -} - -static inline -void subbuffer_inc_packet_count(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) -{ - bufb->buf_cnt[idx].seq_cnt++; -} - -/** - * lib_ring_buffer_clear_noref - Clear the noref subbuffer flag, called by - * writer. - */ -static inline -void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx) -{ - unsigned long id, new_id; - - if (config->mode != RING_BUFFER_OVERWRITE) - return; - - /* - * Performing a volatile access to read the sb_pages, because we want to - * read a coherent version of the pointer and the associated noref flag. - */ - id = READ_ONCE(bufb->buf_wsb[idx].id); - for (;;) { - /* This check is called on the fast path for each record. */ - if (likely(!subbuffer_id_is_noref(config, id))) { - /* - * Store after load dependency ordering the writes to - * the subbuffer after load and test of the noref flag - * matches the memory barrier implied by the cmpxchg() - * in update_read_sb_index(). - */ - return; /* Already writing to this buffer */ - } - new_id = id; - subbuffer_id_clear_noref(config, &new_id); - new_id = cmpxchg(&bufb->buf_wsb[idx].id, id, new_id); - if (likely(new_id == id)) - break; - id = new_id; - } -} - -/** - * lib_ring_buffer_set_noref_offset - Set the noref subbuffer flag and offset, - * called by writer. - */ -static inline -void lib_ring_buffer_set_noref_offset(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - unsigned long idx, unsigned long offset) -{ - if (config->mode != RING_BUFFER_OVERWRITE) - return; - - /* - * Because ring_buffer_set_noref() is only called by a single thread - * (the one which updated the cc_sb value), there are no concurrent - * updates to take care of: other writers have not updated cc_sb, so - * they cannot set the noref flag, and concurrent readers cannot modify - * the pointer because the noref flag is not set yet. - * The smp_wmb() in ring_buffer_commit() takes care of ordering writes - * to the subbuffer before this set noref operation. - * subbuffer_set_noref() uses a volatile store to deal with concurrent - * readers of the noref flag. - */ - CHAN_WARN_ON(bufb->chan, - subbuffer_id_is_noref(config, bufb->buf_wsb[idx].id)); - /* - * Memory barrier that ensures counter stores are ordered before set - * noref and offset. - */ - smp_mb(); - subbuffer_id_set_noref_offset(config, &bufb->buf_wsb[idx].id, offset); -} - -/** - * update_read_sb_index - Read-side subbuffer index update. - */ -static inline -int update_read_sb_index(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_backend *bufb, - struct channel_backend *chanb, - unsigned long consumed_idx, - unsigned long consumed_count) -{ - unsigned long old_id, new_id; - - if (config->mode == RING_BUFFER_OVERWRITE) { - /* - * Exchange the target writer subbuffer with our own unused - * subbuffer. No need to use READ_ONCE() here to read the - * old_wpage, because the value read will be confirmed by the - * following cmpxchg(). - */ - old_id = bufb->buf_wsb[consumed_idx].id; - if (unlikely(!subbuffer_id_is_noref(config, old_id))) - return -EAGAIN; - /* - * Make sure the offset count we are expecting matches the one - * indicated by the writer. - */ - if (unlikely(!subbuffer_id_compare_offset(config, old_id, - consumed_count))) - return -EAGAIN; - CHAN_WARN_ON(bufb->chan, - !subbuffer_id_is_noref(config, bufb->buf_rsb.id)); - subbuffer_id_set_noref_offset(config, &bufb->buf_rsb.id, - consumed_count); - new_id = cmpxchg(&bufb->buf_wsb[consumed_idx].id, old_id, - bufb->buf_rsb.id); - if (unlikely(old_id != new_id)) - return -EAGAIN; - bufb->buf_rsb.id = new_id; - } else { - /* No page exchange, use the writer page directly */ - bufb->buf_rsb.id = bufb->buf_wsb[consumed_idx].id; - } - return 0; -} - -static inline __attribute__((always_inline)) -void lttng_inline_memcpy(void *dest, const void *src, - unsigned long len) -{ - switch (len) { - case 1: - *(uint8_t *) dest = *(const uint8_t *) src; - break; - case 2: - *(uint16_t *) dest = *(const uint16_t *) src; - break; - case 4: - *(uint32_t *) dest = *(const uint32_t *) src; - break; - case 8: - *(uint64_t *) dest = *(const uint64_t *) src; - break; - default: - memcpy(dest, src, len); - } -} - -/* - * Use the architecture-specific memcpy implementation for constant-sized - * inputs, but rely on an inline memcpy for length statically unknown. - * The function call to memcpy is just way too expensive for a fast path. - */ -#define lib_ring_buffer_do_copy(config, dest, src, len) \ -do { \ - size_t __len = (len); \ - if (__builtin_constant_p(len)) \ - memcpy(dest, src, __len); \ - else \ - lttng_inline_memcpy(dest, src, __len); \ -} while (0) - -/* - * We use __copy_from_user_inatomic to copy userspace data since we already - * did the access_ok for the whole range. - * - * Return 0 if OK, nonzero on error. - */ -static inline -unsigned long lib_ring_buffer_do_copy_from_user_inatomic(void *dest, - const void __user *src, - unsigned long len) -{ - return __copy_from_user_inatomic(dest, src, len); -} - -/* - * write len bytes to dest with c - */ -static inline -void lib_ring_buffer_do_memset(char *dest, int c, - unsigned long len) -{ - unsigned long i; - - for (i = 0; i < len; i++) - dest[i] = c; -} - -#endif /* _LIB_RING_BUFFER_BACKEND_INTERNAL_H */ diff --git a/lib/ringbuffer/backend_types.h b/lib/ringbuffer/backend_types.h deleted file mode 100644 index 3766d8bd..00000000 --- a/lib/ringbuffer/backend_types.h +++ /dev/null @@ -1,100 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/backend_types.h - * - * Ring buffer backend (types). - * - * Copyright (C) 2008-2012 Mathieu Desnoyers - */ - -#ifndef _LIB_RING_BUFFER_BACKEND_TYPES_H -#define _LIB_RING_BUFFER_BACKEND_TYPES_H - -#include -#include -#include -#include - -struct lib_ring_buffer_backend_page { - void *virt; /* page virtual address (cached) */ - unsigned long pfn; /* page frame number */ -}; - -struct lib_ring_buffer_backend_pages { - unsigned long mmap_offset; /* offset of the subbuffer in mmap */ - union v_atomic records_commit; /* current records committed count */ - union v_atomic records_unread; /* records to read */ - unsigned long data_size; /* Amount of data to read from subbuf */ - struct lib_ring_buffer_backend_page p[]; -}; - -struct lib_ring_buffer_backend_subbuffer { - /* Identifier for subbuf backend pages. Exchanged atomically. */ - unsigned long id; /* backend subbuffer identifier */ -}; - -struct lib_ring_buffer_backend_counts { - /* - * Counter specific to the sub-buffer location within the ring buffer. - * The actual sequence number of the packet within the entire ring - * buffer can be derived from the formula nr_subbuffers * seq_cnt + - * subbuf_idx. - */ - uint64_t seq_cnt; /* packet sequence number */ -}; - -/* - * Forward declaration of frontend-specific channel and ring_buffer. - */ -struct channel; -struct lib_ring_buffer; - -struct lib_ring_buffer_backend { - /* Array of ring_buffer_backend_subbuffer for writer */ - struct lib_ring_buffer_backend_subbuffer *buf_wsb; - /* ring_buffer_backend_subbuffer for reader */ - struct lib_ring_buffer_backend_subbuffer buf_rsb; - /* Array of lib_ring_buffer_backend_counts for the packet counter */ - struct lib_ring_buffer_backend_counts *buf_cnt; - /* - * Pointer array of backend pages, for whole buffer. - * Indexed by ring_buffer_backend_subbuffer identifier (id) index. - */ - struct lib_ring_buffer_backend_pages **array; - unsigned int num_pages_per_subbuf; - - struct channel *chan; /* Associated channel */ - int cpu; /* This buffer's cpu. -1 if global. */ - union v_atomic records_read; /* Number of records read */ - unsigned int allocated:1; /* is buffer allocated ? */ -}; - -struct channel_backend { - unsigned long buf_size; /* Size of the buffer */ - unsigned long subbuf_size; /* Sub-buffer size */ - unsigned int subbuf_size_order; /* Order of sub-buffer size */ - unsigned int num_subbuf_order; /* - * Order of number of sub-buffers/buffer - * for writer. - */ - unsigned int buf_size_order; /* Order of buffer size */ - unsigned int extra_reader_sb:1; /* has extra reader subbuffer ? */ - struct lib_ring_buffer *buf; /* Channel per-cpu buffers */ - - unsigned long num_subbuf; /* Number of sub-buffers for writer */ - u64 start_tsc; /* Channel creation TSC value */ - void *priv; /* Client-specific information */ - void *priv_ops; /* Client-specific ops pointer */ - void (*release_priv_ops)(void *priv_ops); - struct lttng_cpuhp_node cpuhp_prepare; /* CPU hotplug prepare */ - /* - * We need to copy config because the module containing the - * source config can vanish before the last reference to this - * channel's streams is released. - */ - struct lib_ring_buffer_config config; /* Ring buffer configuration */ - cpumask_var_t cpumask; /* Allocated per-cpu buffers cpumask */ - char name[NAME_MAX]; /* Channel name */ -}; - -#endif /* _LIB_RING_BUFFER_BACKEND_TYPES_H */ diff --git a/lib/ringbuffer/config.h b/lib/ringbuffer/config.h deleted file mode 100644 index 15259239..00000000 --- a/lib/ringbuffer/config.h +++ /dev/null @@ -1,306 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/config.h - * - * Ring buffer configuration header. Note: after declaring the standard inline - * functions, clients should also include linux/ringbuffer/api.h. - * - * Copyright (C) 2010-2012 Mathieu Desnoyers - */ - -#ifndef _LIB_RING_BUFFER_CONFIG_H -#define _LIB_RING_BUFFER_CONFIG_H - -#include -#include -#include -#include - -struct lib_ring_buffer; -struct channel; -struct lib_ring_buffer_config; -struct lib_ring_buffer_ctx; - -/* - * Ring buffer client callbacks. Only used by slow path, never on fast path. - * For the fast path, record_header_size(), ring_buffer_clock_read() should be - * provided as inline functions too. These may simply return 0 if not used by - * the client. - */ -struct lib_ring_buffer_client_cb { - /* Mandatory callbacks */ - - /* A static inline version is also required for fast path */ - u64 (*ring_buffer_clock_read) (struct channel *chan); - size_t (*record_header_size) (const struct lib_ring_buffer_config *config, - struct channel *chan, size_t offset, - size_t *pre_header_padding, - struct lib_ring_buffer_ctx *ctx, - void *client_ctx); - - /* Slow path only, at subbuffer switch */ - size_t (*subbuffer_header_size) (void); - void (*buffer_begin) (struct lib_ring_buffer *buf, u64 tsc, - unsigned int subbuf_idx); - void (*buffer_end) (struct lib_ring_buffer *buf, u64 tsc, - unsigned int subbuf_idx, unsigned long data_size); - - /* Optional callbacks (can be set to NULL) */ - - /* Called at buffer creation/finalize */ - int (*buffer_create) (struct lib_ring_buffer *buf, void *priv, - int cpu, const char *name); - /* - * Clients should guarantee that no new reader handle can be opened - * after finalize. - */ - void (*buffer_finalize) (struct lib_ring_buffer *buf, void *priv, int cpu); - - /* - * Extract header length, payload length and timestamp from event - * record. Used by buffer iterators. Timestamp is only used by channel - * iterator. - */ - void (*record_get) (const struct lib_ring_buffer_config *config, - struct channel *chan, struct lib_ring_buffer *buf, - size_t offset, size_t *header_len, - size_t *payload_len, u64 *timestamp); -}; - -/* - * Ring buffer instance configuration. - * - * Declare as "static const" within the client object to ensure the inline fast - * paths can be optimized. - * - * alloc/sync pairs: - * - * RING_BUFFER_ALLOC_PER_CPU and RING_BUFFER_SYNC_PER_CPU : - * Per-cpu buffers with per-cpu synchronization. Tracing must be performed - * with preemption disabled (lib_ring_buffer_get_cpu() and - * lib_ring_buffer_put_cpu()). - * - * RING_BUFFER_ALLOC_PER_CPU and RING_BUFFER_SYNC_GLOBAL : - * Per-cpu buffer with global synchronization. Tracing can be performed with - * preemption enabled, statistically stays on the local buffers. - * - * RING_BUFFER_ALLOC_GLOBAL and RING_BUFFER_SYNC_PER_CPU : - * Should only be used for buffers belonging to a single thread or protected - * by mutual exclusion by the client. Note that periodical sub-buffer switch - * should be disabled in this kind of configuration. - * - * RING_BUFFER_ALLOC_GLOBAL and RING_BUFFER_SYNC_GLOBAL : - * Global shared buffer with global synchronization. - * - * wakeup: - * - * RING_BUFFER_WAKEUP_BY_TIMER uses per-cpu timers to poll the - * buffers and wake up readers if data is ready. Mainly useful for tracers which - * don't want to call into the wakeup code on the tracing path. Use in - * combination with "read_timer_interval" channel_create() argument. - * - * RING_BUFFER_WAKEUP_BY_WRITER directly wakes up readers when a subbuffer is - * ready to read. Lower latencies before the reader is woken up. Mainly suitable - * for drivers. - * - * RING_BUFFER_WAKEUP_NONE does not perform any wakeup whatsoever. The client - * has the responsibility to perform wakeups. - */ -struct lib_ring_buffer_config { - enum { - RING_BUFFER_ALLOC_PER_CPU, - RING_BUFFER_ALLOC_GLOBAL, - } alloc; - enum { - RING_BUFFER_SYNC_PER_CPU, /* Wait-free */ - RING_BUFFER_SYNC_GLOBAL, /* Lock-free */ - } sync; - enum { - RING_BUFFER_OVERWRITE, /* Overwrite when buffer full */ - RING_BUFFER_DISCARD, /* Discard when buffer full */ - } mode; - enum { - RING_BUFFER_SPLICE, - RING_BUFFER_MMAP, - RING_BUFFER_READ, /* TODO */ - RING_BUFFER_ITERATOR, - RING_BUFFER_NONE, - } output; - enum { - RING_BUFFER_PAGE, - RING_BUFFER_VMAP, /* TODO */ - RING_BUFFER_STATIC, /* TODO */ - } backend; - enum { - RING_BUFFER_NO_OOPS_CONSISTENCY, - RING_BUFFER_OOPS_CONSISTENCY, - } oops; - enum { - RING_BUFFER_IPI_BARRIER, - RING_BUFFER_NO_IPI_BARRIER, - } ipi; - enum { - RING_BUFFER_WAKEUP_BY_TIMER, /* wake up performed by timer */ - RING_BUFFER_WAKEUP_BY_WRITER, /* - * writer wakes up reader, - * not lock-free - * (takes spinlock). - */ - } wakeup; - /* - * tsc_bits: timestamp bits saved at each record. - * 0 and 64 disable the timestamp compression scheme. - */ - unsigned int tsc_bits; - struct lib_ring_buffer_client_cb cb; -}; - -/* - * ring buffer context - * - * Context passed to lib_ring_buffer_reserve(), lib_ring_buffer_commit(), - * lib_ring_buffer_try_discard_reserve(), lib_ring_buffer_align_ctx() and - * lib_ring_buffer_write(). - */ -struct lib_ring_buffer_ctx { - /* input received by lib_ring_buffer_reserve(), saved here. */ - struct channel *chan; /* channel */ - void *priv; /* client private data */ - size_t data_size; /* size of payload */ - int largest_align; /* - * alignment of the largest element - * in the payload - */ - int cpu; /* processor id */ - - /* output from lib_ring_buffer_reserve() */ - struct lib_ring_buffer *buf; /* - * buffer corresponding to processor id - * for this channel - */ - size_t slot_size; /* size of the reserved slot */ - unsigned long buf_offset; /* offset following the record header */ - unsigned long pre_offset; /* - * Initial offset position _before_ - * the record is written. Positioned - * prior to record header alignment - * padding. - */ - u64 tsc; /* time-stamp counter value */ - unsigned int rflags; /* reservation flags */ - /* Cache backend pages pointer chasing. */ - struct lib_ring_buffer_backend_pages *backend_pages; -}; - -/** - * lib_ring_buffer_ctx_init - initialize ring buffer context - * @ctx: ring buffer context to initialize - * @chan: channel - * @priv: client private data - * @data_size: size of record data payload. It must be greater than 0. - * @largest_align: largest alignment within data payload types - * @cpu: processor id - */ -static inline -void lib_ring_buffer_ctx_init(struct lib_ring_buffer_ctx *ctx, - struct channel *chan, void *priv, - size_t data_size, int largest_align, - int cpu) -{ - ctx->chan = chan; - ctx->priv = priv; - ctx->data_size = data_size; - ctx->largest_align = largest_align; - ctx->cpu = cpu; - ctx->rflags = 0; - ctx->backend_pages = NULL; -} - -/* - * Reservation flags. - * - * RING_BUFFER_RFLAG_FULL_TSC - * - * This flag is passed to record_header_size() and to the primitive used to - * write the record header. It indicates that the full 64-bit time value is - * needed in the record header. If this flag is not set, the record header needs - * only to contain "tsc_bits" bit of time value. - * - * Reservation flags can be added by the client, starting from - * "(RING_BUFFER_FLAGS_END << 0)". It can be used to pass information from - * record_header_size() to lib_ring_buffer_write_record_header(). - */ -#define RING_BUFFER_RFLAG_FULL_TSC (1U << 0) -#define RING_BUFFER_RFLAG_END (1U << 1) - -#ifndef LTTNG_TRACER_CORE_H -#error "lttng-tracer-core.h is needed for RING_BUFFER_ALIGN define" -#endif - -/* - * We need to define RING_BUFFER_ALIGN_ATTR so it is known early at - * compile-time. We have to duplicate the "config->align" information and the - * definition here because config->align is used both in the slow and fast - * paths, but RING_BUFFER_ALIGN_ATTR is only available for the client code. - */ -#ifdef RING_BUFFER_ALIGN - -# define RING_BUFFER_ALIGN_ATTR /* Default arch alignment */ - -/* - * Calculate the offset needed to align the type. - * size_of_type must be non-zero. - */ -static inline -unsigned int lib_ring_buffer_align(size_t align_drift, size_t size_of_type) -{ - return offset_align(align_drift, size_of_type); -} - -#else - -# define RING_BUFFER_ALIGN_ATTR __attribute__((packed)) - -/* - * Calculate the offset needed to align the type. - * size_of_type must be non-zero. - */ -static inline -unsigned int lib_ring_buffer_align(size_t align_drift, size_t size_of_type) -{ - return 0; -} - -#endif - -/** - * lib_ring_buffer_align_ctx - Align context offset on "alignment" - * @ctx: ring buffer context. - */ -static inline -void lib_ring_buffer_align_ctx(struct lib_ring_buffer_ctx *ctx, - size_t alignment) -{ - ctx->buf_offset += lib_ring_buffer_align(ctx->buf_offset, - alignment); -} - -/* - * lib_ring_buffer_check_config() returns 0 on success. - * Used internally to check for valid configurations at channel creation. - */ -static inline -int lib_ring_buffer_check_config(const struct lib_ring_buffer_config *config, - unsigned int switch_timer_interval, - unsigned int read_timer_interval) -{ - if (config->alloc == RING_BUFFER_ALLOC_GLOBAL - && config->sync == RING_BUFFER_SYNC_PER_CPU - && switch_timer_interval) - return -EINVAL; - return 0; -} - -#include - -#endif /* _LIB_RING_BUFFER_CONFIG_H */ diff --git a/lib/ringbuffer/frontend.h b/lib/ringbuffer/frontend.h deleted file mode 100644 index ae1c1aaf..00000000 --- a/lib/ringbuffer/frontend.h +++ /dev/null @@ -1,234 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/frontend.h - * - * Ring Buffer Library Synchronization Header (API). - * - * Copyright (C) 2005-2012 Mathieu Desnoyers - * - * See ring_buffer_frontend.c for more information on wait-free algorithms. - */ - -#ifndef _LIB_RING_BUFFER_FRONTEND_H -#define _LIB_RING_BUFFER_FRONTEND_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* Internal helpers */ -#include - -/* Max ring buffer nesting count, see lib_ring_buffer_get_cpu(). */ -#define RING_BUFFER_MAX_NESTING 4 - -/* Buffer creation/removal and setup operations */ - -/* - * switch_timer_interval is the time interval (in us) to fill sub-buffers with - * padding to let readers get those sub-buffers. Used for live streaming. - * - * read_timer_interval is the time interval (in us) to wake up pending readers. - * - * buf_addr is a pointer the the beginning of the preallocated buffer contiguous - * address mapping. It is used only by RING_BUFFER_STATIC configuration. It can - * be set to NULL for other backends. - */ - -extern -struct channel *channel_create(const struct lib_ring_buffer_config *config, - const char *name, void *priv, - void *buf_addr, - size_t subbuf_size, size_t num_subbuf, - unsigned int switch_timer_interval, - unsigned int read_timer_interval); - -/* - * channel_destroy returns the private data pointer. It finalizes all channel's - * buffers, waits for readers to release all references, and destroys the - * channel. - */ -extern -void *channel_destroy(struct channel *chan); - - -/* Buffer read operations */ - -/* - * Iteration on channel cpumask needs to issue a read barrier to match the write - * barrier in cpu hotplug. It orders the cpumask read before read of per-cpu - * buffer data. The per-cpu buffer is never removed by cpu hotplug; teardown is - * only performed at channel destruction. - */ -#define for_each_channel_cpu(cpu, chan) \ - for ((cpu) = -1; \ - ({ (cpu) = cpumask_next(cpu, (chan)->backend.cpumask); \ - smp_read_barrier_depends(); (cpu) < nr_cpu_ids; });) - -extern struct lib_ring_buffer *channel_get_ring_buffer( - const struct lib_ring_buffer_config *config, - struct channel *chan, int cpu); -extern int lib_ring_buffer_open_read(struct lib_ring_buffer *buf); -extern void lib_ring_buffer_release_read(struct lib_ring_buffer *buf); - -/* - * Read sequence: snapshot, many get_subbuf/put_subbuf, move_consumer. - */ -extern int lib_ring_buffer_snapshot(struct lib_ring_buffer *buf, - unsigned long *consumed, - unsigned long *produced); -extern int lib_ring_buffer_snapshot_sample_positions( - struct lib_ring_buffer *buf, - unsigned long *consumed, - unsigned long *produced); -extern void lib_ring_buffer_move_consumer(struct lib_ring_buffer *buf, - unsigned long consumed_new); - -extern int lib_ring_buffer_get_subbuf(struct lib_ring_buffer *buf, - unsigned long consumed); -extern void lib_ring_buffer_put_subbuf(struct lib_ring_buffer *buf); - -void lib_ring_buffer_set_quiescent_channel(struct channel *chan); -void lib_ring_buffer_clear_quiescent_channel(struct channel *chan); - -/* - * lib_ring_buffer_get_next_subbuf/lib_ring_buffer_put_next_subbuf are helpers - * to read sub-buffers sequentially. - */ -static inline int lib_ring_buffer_get_next_subbuf(struct lib_ring_buffer *buf) -{ - int ret; - - ret = lib_ring_buffer_snapshot(buf, &buf->cons_snapshot, - &buf->prod_snapshot); - if (ret) - return ret; - ret = lib_ring_buffer_get_subbuf(buf, buf->cons_snapshot); - return ret; -} - -static inline void lib_ring_buffer_put_next_subbuf(struct lib_ring_buffer *buf) -{ - lib_ring_buffer_put_subbuf(buf); - lib_ring_buffer_move_consumer(buf, subbuf_align(buf->cons_snapshot, - buf->backend.chan)); -} - -extern void channel_reset(struct channel *chan); -extern void lib_ring_buffer_reset(struct lib_ring_buffer *buf); - -static inline -unsigned long lib_ring_buffer_get_offset(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - return v_read(config, &buf->offset); -} - -static inline -unsigned long lib_ring_buffer_get_consumed(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - return atomic_long_read(&buf->consumed); -} - -/* - * Must call lib_ring_buffer_is_finalized before reading counters (memory - * ordering enforced with respect to trace teardown). - */ -static inline -int lib_ring_buffer_is_finalized(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - int finalized = READ_ONCE(buf->finalized); - /* - * Read finalized before counters. - */ - smp_rmb(); - return finalized; -} - -static inline -int lib_ring_buffer_channel_is_finalized(const struct channel *chan) -{ - return chan->finalized; -} - -static inline -int lib_ring_buffer_channel_is_disabled(const struct channel *chan) -{ - return atomic_read(&chan->record_disabled); -} - -static inline -unsigned long lib_ring_buffer_get_read_data_size( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - return subbuffer_get_read_data_size(config, &buf->backend); -} - -static inline -unsigned long lib_ring_buffer_get_records_count( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - return v_read(config, &buf->records_count); -} - -static inline -unsigned long lib_ring_buffer_get_records_overrun( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - return v_read(config, &buf->records_overrun); -} - -static inline -unsigned long lib_ring_buffer_get_records_lost_full( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - return v_read(config, &buf->records_lost_full); -} - -static inline -unsigned long lib_ring_buffer_get_records_lost_wrap( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - return v_read(config, &buf->records_lost_wrap); -} - -static inline -unsigned long lib_ring_buffer_get_records_lost_big( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - return v_read(config, &buf->records_lost_big); -} - -static inline -unsigned long lib_ring_buffer_get_records_read( - const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - return v_read(config, &buf->backend.records_read); -} - -#endif /* _LIB_RING_BUFFER_FRONTEND_H */ diff --git a/lib/ringbuffer/frontend_api.h b/lib/ringbuffer/frontend_api.h deleted file mode 100644 index 252aedc2..00000000 --- a/lib/ringbuffer/frontend_api.h +++ /dev/null @@ -1,358 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/frontend_api.h - * - * Ring Buffer Library Synchronization Header (buffer write API). - * - * Copyright (C) 2005-2012 Mathieu Desnoyers - * - * See ring_buffer_frontend.c for more information on wait-free algorithms. - * See linux/ringbuffer/frontend.h for channel allocation and read-side API. - */ - -#ifndef _LIB_RING_BUFFER_FRONTEND_API_H -#define _LIB_RING_BUFFER_FRONTEND_API_H - -#include -#include -#include -#include - -/** - * lib_ring_buffer_get_cpu - Precedes ring buffer reserve/commit. - * - * Disables preemption (acts as a RCU read-side critical section) and keeps a - * ring buffer nesting count as supplementary safety net to ensure tracer client - * code will never trigger an endless recursion. Returns the processor ID on - * success, -EPERM on failure (nesting count too high). - * - * asm volatile and "memory" clobber prevent the compiler from moving - * instructions out of the ring buffer nesting count. This is required to ensure - * that probe side-effects which can cause recursion (e.g. unforeseen traps, - * divisions by 0, ...) are triggered within the incremented nesting count - * section. - */ -static inline -int lib_ring_buffer_get_cpu(const struct lib_ring_buffer_config *config) -{ - int cpu, nesting; - - rcu_read_lock_sched_notrace(); - cpu = smp_processor_id(); - nesting = ++per_cpu(lib_ring_buffer_nesting, cpu); - barrier(); - - if (unlikely(nesting > RING_BUFFER_MAX_NESTING)) { - WARN_ON_ONCE(1); - per_cpu(lib_ring_buffer_nesting, cpu)--; - rcu_read_unlock_sched_notrace(); - return -EPERM; - } else - return cpu; -} - -/** - * lib_ring_buffer_put_cpu - Follows ring buffer reserve/commit. - */ -static inline -void lib_ring_buffer_put_cpu(const struct lib_ring_buffer_config *config) -{ - barrier(); - (*this_cpu_ptr(&lib_ring_buffer_nesting))--; - rcu_read_unlock_sched_notrace(); -} - -/* - * lib_ring_buffer_try_reserve is called by lib_ring_buffer_reserve(). It is not - * part of the API per se. - * - * returns 0 if reserve ok, or 1 if the slow path must be taken. - */ -static inline -int lib_ring_buffer_try_reserve(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_ctx *ctx, - void *client_ctx, - unsigned long *o_begin, unsigned long *o_end, - unsigned long *o_old, size_t *before_hdr_pad) -{ - struct channel *chan = ctx->chan; - struct lib_ring_buffer *buf = ctx->buf; - *o_begin = v_read(config, &buf->offset); - *o_old = *o_begin; - - ctx->tsc = lib_ring_buffer_clock_read(chan); - if ((int64_t) ctx->tsc == -EIO) - return 1; - - /* - * Prefetch cacheline for read because we have to read the previous - * commit counter to increment it and commit seq value to compare it to - * the commit counter. - */ - prefetch(&buf->commit_hot[subbuf_index(*o_begin, chan)]); - - if (last_tsc_overflow(config, buf, ctx->tsc)) - ctx->rflags |= RING_BUFFER_RFLAG_FULL_TSC; - - if (unlikely(subbuf_offset(*o_begin, chan) == 0)) - return 1; - - ctx->slot_size = record_header_size(config, chan, *o_begin, - before_hdr_pad, ctx, client_ctx); - ctx->slot_size += - lib_ring_buffer_align(*o_begin + ctx->slot_size, - ctx->largest_align) + ctx->data_size; - if (unlikely((subbuf_offset(*o_begin, chan) + ctx->slot_size) - > chan->backend.subbuf_size)) - return 1; - - /* - * Record fits in the current buffer and we are not on a switch - * boundary. It's safe to write. - */ - *o_end = *o_begin + ctx->slot_size; - - if (unlikely((subbuf_offset(*o_end, chan)) == 0)) - /* - * The offset_end will fall at the very beginning of the next - * subbuffer. - */ - return 1; - - return 0; -} - -/** - * lib_ring_buffer_reserve - Reserve space in a ring buffer. - * @config: ring buffer instance configuration. - * @ctx: ring buffer context. (input and output) Must be already initialized. - * - * Atomic wait-free slot reservation. The reserved space starts at the context - * "pre_offset". Its length is "slot_size". The associated time-stamp is "tsc". - * - * Return : - * 0 on success. - * -EAGAIN if channel is disabled. - * -ENOSPC if event size is too large for packet. - * -ENOBUFS if there is currently not enough space in buffer for the event. - * -EIO if data cannot be written into the buffer for any other reason. - */ - -static inline -int lib_ring_buffer_reserve(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer_ctx *ctx, - void *client_ctx) -{ - struct channel *chan = ctx->chan; - struct lib_ring_buffer *buf; - unsigned long o_begin, o_end, o_old; - size_t before_hdr_pad = 0; - - if (unlikely(atomic_read(&chan->record_disabled))) - return -EAGAIN; - - if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) - buf = per_cpu_ptr(chan->backend.buf, ctx->cpu); - else - buf = chan->backend.buf; - if (unlikely(atomic_read(&buf->record_disabled))) - return -EAGAIN; - ctx->buf = buf; - - /* - * Perform retryable operations. - */ - if (unlikely(lib_ring_buffer_try_reserve(config, ctx, client_ctx, &o_begin, - &o_end, &o_old, &before_hdr_pad))) - goto slow_path; - - if (unlikely(v_cmpxchg(config, &ctx->buf->offset, o_old, o_end) - != o_old)) - goto slow_path; - - /* - * Atomically update last_tsc. This update races against concurrent - * atomic updates, but the race will always cause supplementary full TSC - * record headers, never the opposite (missing a full TSC record header - * when it would be needed). - */ - save_last_tsc(config, ctx->buf, ctx->tsc); - - /* - * Push the reader if necessary - */ - lib_ring_buffer_reserve_push_reader(ctx->buf, chan, o_end - 1); - - /* - * Clear noref flag for this subbuffer. - */ - lib_ring_buffer_clear_noref(config, &ctx->buf->backend, - subbuf_index(o_end - 1, chan)); - - ctx->pre_offset = o_begin; - ctx->buf_offset = o_begin + before_hdr_pad; - return 0; -slow_path: - return lib_ring_buffer_reserve_slow(ctx, client_ctx); -} - -/** - * lib_ring_buffer_switch - Perform a sub-buffer switch for a per-cpu buffer. - * @config: ring buffer instance configuration. - * @buf: buffer - * @mode: buffer switch mode (SWITCH_ACTIVE or SWITCH_FLUSH) - * - * This operation is completely reentrant : can be called while tracing is - * active with absolutely no lock held. - * - * Note, however, that as a v_cmpxchg is used for some atomic operations and - * requires to be executed locally for per-CPU buffers, this function must be - * called from the CPU which owns the buffer for a ACTIVE flush, with preemption - * disabled, for RING_BUFFER_SYNC_PER_CPU configuration. - */ -static inline -void lib_ring_buffer_switch(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, enum switch_mode mode) -{ - lib_ring_buffer_switch_slow(buf, mode); -} - -/* See ring_buffer_frontend_api.h for lib_ring_buffer_reserve(). */ - -/** - * lib_ring_buffer_commit - Commit an record. - * @config: ring buffer instance configuration. - * @ctx: ring buffer context. (input arguments only) - * - * Atomic unordered slot commit. Increments the commit count in the - * specified sub-buffer, and delivers it if necessary. - */ -static inline -void lib_ring_buffer_commit(const struct lib_ring_buffer_config *config, - const struct lib_ring_buffer_ctx *ctx) -{ - struct channel *chan = ctx->chan; - struct lib_ring_buffer *buf = ctx->buf; - unsigned long offset_end = ctx->buf_offset; - unsigned long endidx = subbuf_index(offset_end - 1, chan); - unsigned long commit_count; - struct commit_counters_hot *cc_hot = &buf->commit_hot[endidx]; - - /* - * Must count record before incrementing the commit count. - */ - subbuffer_count_record(config, &buf->backend, endidx); - - /* - * Order all writes to buffer before the commit count update that will - * determine that the subbuffer is full. - */ - if (config->ipi == RING_BUFFER_IPI_BARRIER) { - /* - * Must write slot data before incrementing commit count. This - * compiler barrier is upgraded into a smp_mb() by the IPI sent - * by get_subbuf(). - */ - barrier(); - } else - smp_wmb(); - - v_add(config, ctx->slot_size, &cc_hot->cc); - - /* - * commit count read can race with concurrent OOO commit count updates. - * This is only needed for lib_ring_buffer_check_deliver (for - * non-polling delivery only) and for - * lib_ring_buffer_write_commit_counter. The race can only cause the - * counter to be read with the same value more than once, which could - * cause : - * - Multiple delivery for the same sub-buffer (which is handled - * gracefully by the reader code) if the value is for a full - * sub-buffer. It's important that we can never miss a sub-buffer - * delivery. Re-reading the value after the v_add ensures this. - * - Reading a commit_count with a higher value that what was actually - * added to it for the lib_ring_buffer_write_commit_counter call - * (again caused by a concurrent committer). It does not matter, - * because this function is interested in the fact that the commit - * count reaches back the reserve offset for a specific sub-buffer, - * which is completely independent of the order. - */ - commit_count = v_read(config, &cc_hot->cc); - - lib_ring_buffer_check_deliver(config, buf, chan, offset_end - 1, - commit_count, endidx, ctx->tsc); - /* - * Update used size at each commit. It's needed only for extracting - * ring_buffer buffers from vmcore, after crash. - */ - lib_ring_buffer_write_commit_counter(config, buf, chan, - offset_end, commit_count, cc_hot); -} - -/** - * lib_ring_buffer_try_discard_reserve - Try discarding a record. - * @config: ring buffer instance configuration. - * @ctx: ring buffer context. (input arguments only) - * - * Only succeeds if no other record has been written after the record to - * discard. If discard fails, the record must be committed to the buffer. - * - * Returns 0 upon success, -EPERM if the record cannot be discarded. - */ -static inline -int lib_ring_buffer_try_discard_reserve(const struct lib_ring_buffer_config *config, - const struct lib_ring_buffer_ctx *ctx) -{ - struct lib_ring_buffer *buf = ctx->buf; - unsigned long end_offset = ctx->pre_offset + ctx->slot_size; - - /* - * We need to ensure that if the cmpxchg succeeds and discards the - * record, the next record will record a full TSC, because it cannot - * rely on the last_tsc associated with the discarded record to detect - * overflows. The only way to ensure this is to set the last_tsc to 0 - * (assuming no 64-bit TSC overflow), which forces to write a 64-bit - * timestamp in the next record. - * - * Note: if discard fails, we must leave the TSC in the record header. - * It is needed to keep track of TSC overflows for the following - * records. - */ - save_last_tsc(config, buf, 0ULL); - - if (likely(v_cmpxchg(config, &buf->offset, end_offset, ctx->pre_offset) - != end_offset)) - return -EPERM; - else - return 0; -} - -static inline -void channel_record_disable(const struct lib_ring_buffer_config *config, - struct channel *chan) -{ - atomic_inc(&chan->record_disabled); -} - -static inline -void channel_record_enable(const struct lib_ring_buffer_config *config, - struct channel *chan) -{ - atomic_dec(&chan->record_disabled); -} - -static inline -void lib_ring_buffer_record_disable(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - atomic_inc(&buf->record_disabled); -} - -static inline -void lib_ring_buffer_record_enable(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf) -{ - atomic_dec(&buf->record_disabled); -} - -#endif /* _LIB_RING_BUFFER_FRONTEND_API_H */ diff --git a/lib/ringbuffer/frontend_internal.h b/lib/ringbuffer/frontend_internal.h deleted file mode 100644 index eb17c5da..00000000 --- a/lib/ringbuffer/frontend_internal.h +++ /dev/null @@ -1,334 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * linux/ringbuffer/frontend_internal.h - * - * Ring Buffer Library Synchronization Header (internal helpers). - * - * Copyright (C) 2005-2012 Mathieu Desnoyers - * - * See ring_buffer_frontend.c for more information on wait-free algorithms. - */ - -#ifndef _LIB_RING_BUFFER_FRONTEND_INTERNAL_H -#define _LIB_RING_BUFFER_FRONTEND_INTERNAL_H - -#include -#include -#include -#include /* For per-CPU read-side iterator */ - -/* Buffer offset macros */ - -/* buf_trunc mask selects only the buffer number. */ -static inline -unsigned long buf_trunc(unsigned long offset, struct channel *chan) -{ - return offset & ~(chan->backend.buf_size - 1); - -} - -/* Select the buffer number value (counter). */ -static inline -unsigned long buf_trunc_val(unsigned long offset, struct channel *chan) -{ - return buf_trunc(offset, chan) >> chan->backend.buf_size_order; -} - -/* buf_offset mask selects only the offset within the current buffer. */ -static inline -unsigned long buf_offset(unsigned long offset, struct channel *chan) -{ - return offset & (chan->backend.buf_size - 1); -} - -/* subbuf_offset mask selects the offset within the current subbuffer. */ -static inline -unsigned long subbuf_offset(unsigned long offset, struct channel *chan) -{ - return offset & (chan->backend.subbuf_size - 1); -} - -/* subbuf_trunc mask selects the subbuffer number. */ -static inline -unsigned long subbuf_trunc(unsigned long offset, struct channel *chan) -{ - return offset & ~(chan->backend.subbuf_size - 1); -} - -/* subbuf_align aligns the offset to the next subbuffer. */ -static inline -unsigned long subbuf_align(unsigned long offset, struct channel *chan) -{ - return (offset + chan->backend.subbuf_size) - & ~(chan->backend.subbuf_size - 1); -} - -/* subbuf_index returns the index of the current subbuffer within the buffer. */ -static inline -unsigned long subbuf_index(unsigned long offset, struct channel *chan) -{ - return buf_offset(offset, chan) >> chan->backend.subbuf_size_order; -} - -/* - * Last TSC comparison functions. Check if the current TSC overflows tsc_bits - * bits from the last TSC read. When overflows are detected, the full 64-bit - * timestamp counter should be written in the record header. Reads and writes - * last_tsc atomically. - */ - -#if (BITS_PER_LONG == 32) -static inline -void save_last_tsc(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, u64 tsc) -{ - if (config->tsc_bits == 0 || config->tsc_bits == 64) - return; - - /* - * Ensure the compiler performs this update in a single instruction. - */ - v_set(config, &buf->last_tsc, (unsigned long)(tsc >> config->tsc_bits)); -} - -static inline -int last_tsc_overflow(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, u64 tsc) -{ - unsigned long tsc_shifted; - - if (config->tsc_bits == 0 || config->tsc_bits == 64) - return 0; - - tsc_shifted = (unsigned long)(tsc >> config->tsc_bits); - if (unlikely(tsc_shifted - - (unsigned long)v_read(config, &buf->last_tsc))) - return 1; - else - return 0; -} -#else -static inline -void save_last_tsc(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, u64 tsc) -{ - if (config->tsc_bits == 0 || config->tsc_bits == 64) - return; - - v_set(config, &buf->last_tsc, (unsigned long)tsc); -} - -static inline -int last_tsc_overflow(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, u64 tsc) -{ - if (config->tsc_bits == 0 || config->tsc_bits == 64) - return 0; - - if (unlikely((tsc - v_read(config, &buf->last_tsc)) - >> config->tsc_bits)) - return 1; - else - return 0; -} -#endif - -extern -int lib_ring_buffer_reserve_slow(struct lib_ring_buffer_ctx *ctx, - void *client_ctx); - -extern -void lib_ring_buffer_switch_slow(struct lib_ring_buffer *buf, - enum switch_mode mode); - -extern -void lib_ring_buffer_check_deliver_slow(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, - struct channel *chan, - unsigned long offset, - unsigned long commit_count, - unsigned long idx, - u64 tsc); - -extern -void lib_ring_buffer_switch_remote(struct lib_ring_buffer *buf); -extern -void lib_ring_buffer_switch_remote_empty(struct lib_ring_buffer *buf); -extern -void lib_ring_buffer_clear(struct lib_ring_buffer *buf); - -/* Buffer write helpers */ - -static inline -void lib_ring_buffer_reserve_push_reader(struct lib_ring_buffer *buf, - struct channel *chan, - unsigned long offset) -{ - unsigned long consumed_old, consumed_new; - - do { - consumed_old = atomic_long_read(&buf->consumed); - /* - * If buffer is in overwrite mode, push the reader consumed - * count if the write position has reached it and we are not - * at the first iteration (don't push the reader farther than - * the writer). This operation can be done concurrently by many - * writers in the same buffer, the writer being at the farthest - * write position sub-buffer index in the buffer being the one - * which will win this loop. - */ - if (unlikely(subbuf_trunc(offset, chan) - - subbuf_trunc(consumed_old, chan) - >= chan->backend.buf_size)) - consumed_new = subbuf_align(consumed_old, chan); - else - return; - } while (unlikely(atomic_long_cmpxchg(&buf->consumed, consumed_old, - consumed_new) != consumed_old)); -} - -/* - * Move consumed position to the beginning of subbuffer in which the - * write offset is. Should only be used on ring buffers that are not - * actively being written into, because clear_reader does not take into - * account the commit counters when moving the consumed position, which - * can make concurrent trace producers or consumers observe consumed - * position further than the write offset, which breaks ring buffer - * algorithm guarantees. - */ -static inline -void lib_ring_buffer_clear_reader(struct lib_ring_buffer *buf, - struct channel *chan) -{ - const struct lib_ring_buffer_config *config = &chan->backend.config; - unsigned long offset, consumed_old, consumed_new; - - do { - offset = v_read(config, &buf->offset); - consumed_old = atomic_long_read(&buf->consumed); - CHAN_WARN_ON(chan, (long) (subbuf_trunc(offset, chan) - - subbuf_trunc(consumed_old, chan)) - < 0); - consumed_new = subbuf_trunc(offset, chan); - } while (unlikely(atomic_long_cmpxchg(&buf->consumed, consumed_old, - consumed_new) != consumed_old)); -} - -static inline -int lib_ring_buffer_pending_data(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, - struct channel *chan) -{ - return !!subbuf_offset(v_read(config, &buf->offset), chan); -} - -static inline -unsigned long lib_ring_buffer_get_data_size(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, - unsigned long idx) -{ - return subbuffer_get_data_size(config, &buf->backend, idx); -} - -/* - * Check if all space reservation in a buffer have been committed. This helps - * knowing if an execution context is nested (for per-cpu buffers only). - * This is a very specific ftrace use-case, so we keep this as "internal" API. - */ -static inline -int lib_ring_buffer_reserve_committed(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, - struct channel *chan) -{ - unsigned long offset, idx, commit_count; - - CHAN_WARN_ON(chan, config->alloc != RING_BUFFER_ALLOC_PER_CPU); - CHAN_WARN_ON(chan, config->sync != RING_BUFFER_SYNC_PER_CPU); - - /* - * Read offset and commit count in a loop so they are both read - * atomically wrt interrupts. By deal with interrupt concurrency by - * restarting both reads if the offset has been pushed. Note that given - * we only have to deal with interrupt concurrency here, an interrupt - * modifying the commit count will also modify "offset", so it is safe - * to only check for offset modifications. - */ - do { - offset = v_read(config, &buf->offset); - idx = subbuf_index(offset, chan); - commit_count = v_read(config, &buf->commit_hot[idx].cc); - } while (offset != v_read(config, &buf->offset)); - - return ((buf_trunc(offset, chan) >> chan->backend.num_subbuf_order) - - (commit_count & chan->commit_count_mask) == 0); -} - -/* - * Receive end of subbuffer TSC as parameter. It has been read in the - * space reservation loop of either reserve or switch, which ensures it - * progresses monotonically with event records in the buffer. Therefore, - * it ensures that the end timestamp of a subbuffer is <= begin - * timestamp of the following subbuffers. - */ -static inline -void lib_ring_buffer_check_deliver(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, - struct channel *chan, - unsigned long offset, - unsigned long commit_count, - unsigned long idx, - u64 tsc) -{ - unsigned long old_commit_count = commit_count - - chan->backend.subbuf_size; - - /* Check if all commits have been done */ - if (unlikely((buf_trunc(offset, chan) >> chan->backend.num_subbuf_order) - - (old_commit_count & chan->commit_count_mask) == 0)) - lib_ring_buffer_check_deliver_slow(config, buf, chan, offset, - commit_count, idx, tsc); -} - -/* - * lib_ring_buffer_write_commit_counter - * - * For flight recording. must be called after commit. - * This function increments the subbuffer's commit_seq counter each time the - * commit count reaches back the reserve offset (modulo subbuffer size). It is - * useful for crash dump. - */ -static inline -void lib_ring_buffer_write_commit_counter(const struct lib_ring_buffer_config *config, - struct lib_ring_buffer *buf, - struct channel *chan, - unsigned long buf_offset, - unsigned long commit_count, - struct commit_counters_hot *cc_hot) -{ - unsigned long commit_seq_old; - - if (config->oops != RING_BUFFER_OOPS_CONSISTENCY) - return; - - /* - * subbuf_offset includes commit_count_mask. We can simply - * compare the offsets within the subbuffer without caring about - * buffer full/empty mismatch because offset is never zero here - * (subbuffer header and record headers have non-zero length). - */ - if (unlikely(subbuf_offset(buf_offset - commit_count, chan))) - return; - - commit_seq_old = v_read(config, &cc_hot->seq); - if (likely((long) (commit_seq_old - commit_count) < 0)) - v_set(config, &cc_hot->seq, commit_count); -} - -extern int lib_ring_buffer_create(struct lib_ring_buffer *buf, - struct channel_backend *chanb, int cpu); -extern void lib_ring_buffer_free(struct lib_ring_buffer *buf); - -/* Keep track of trap nesting inside ring buffer code */ -DECLARE_PER_CPU(unsigned int, lib_ring_buffer_nesting); - -#endif /* _LIB_RING_BUFFER_FRONTEND_INTERNAL_H */ diff --git a/lib/ringbuffer/frontend_types.h b/lib/ringbuffer/frontend_types.h deleted file mode 100644 index 75e602f4..00000000 --- a/lib/ringbuffer/frontend_types.h +++ /dev/null @@ -1,188 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/frontend_types.h - * - * Ring Buffer Library Synchronization Header (types). - * - * Copyright (C) 2010-2012 Mathieu Desnoyers - * - * See ring_buffer_frontend.c for more information on wait-free algorithms. - */ - -#ifndef _LIB_RING_BUFFER_FRONTEND_TYPES_H -#define _LIB_RING_BUFFER_FRONTEND_TYPES_H - -#include -#include -#include -#include /* For per-CPU read-side iterator */ -#include - -/* - * A switch is done during tracing or as a final flush after tracing (so it - * won't write in the new sub-buffer). - */ -enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH }; - -/* channel-level read-side iterator */ -struct channel_iter { - /* Prio heap of buffers. Lowest timestamps at the top. */ - struct lttng_ptr_heap heap; /* Heap of struct lib_ring_buffer ptrs */ - struct list_head empty_head; /* Empty buffers linked-list head */ - int read_open; /* Opened for reading ? */ - u64 last_qs; /* Last quiescent state timestamp */ - u64 last_timestamp; /* Last timestamp (for WARN_ON) */ - int last_cpu; /* Last timestamp cpu */ - /* - * read() file operation state. - */ - unsigned long len_left; -}; - -/* channel: collection of per-cpu ring buffers. */ -struct channel { - atomic_t record_disabled; - unsigned long commit_count_mask; /* - * Commit count mask, removing - * the MSBs corresponding to - * bits used to represent the - * subbuffer index. - */ - - struct channel_backend backend; /* Associated backend */ - - unsigned long switch_timer_interval; /* Buffer flush (jiffies) */ - unsigned long read_timer_interval; /* Reader wakeup (jiffies) */ - struct lttng_cpuhp_node cpuhp_prepare; - struct lttng_cpuhp_node cpuhp_online; - struct lttng_cpuhp_node cpuhp_iter_online; - struct notifier_block tick_nohz_notifier; /* CPU nohz notifier */ - wait_queue_head_t read_wait; /* reader wait queue */ - wait_queue_head_t hp_wait; /* CPU hotplug wait queue */ - int finalized; /* Has channel been finalized */ - struct channel_iter iter; /* Channel read-side iterator */ - struct kref ref; /* Reference count */ -}; - -/* Per-subbuffer commit counters used on the hot path */ -struct commit_counters_hot { - union v_atomic cc; /* Commit counter */ - union v_atomic seq; /* Consecutive commits */ -}; - -/* Per-subbuffer commit counters used only on cold paths */ -struct commit_counters_cold { - union v_atomic cc_sb; /* Incremented _once_ at sb switch */ -}; - -/* Per-buffer read iterator */ -struct lib_ring_buffer_iter { - u64 timestamp; /* Current record timestamp */ - size_t header_len; /* Current record header length */ - size_t payload_len; /* Current record payload length */ - - struct list_head empty_node; /* Linked list of empty buffers */ - unsigned long consumed, read_offset, data_size; - enum { - ITER_GET_SUBBUF = 0, - ITER_TEST_RECORD, - ITER_NEXT_RECORD, - ITER_PUT_SUBBUF, - } state; - unsigned int allocated:1; - unsigned int read_open:1; /* Opened for reading ? */ -}; - -/* ring buffer state */ -struct lib_ring_buffer { - /* First 32 bytes cache-hot cacheline */ - union v_atomic offset; /* Current offset in the buffer */ - struct commit_counters_hot *commit_hot; - /* Commit count per sub-buffer */ - atomic_long_t consumed; /* - * Current offset in the buffer - * standard atomic access (shared) - */ - atomic_t record_disabled; - /* End of first 32 bytes cacheline */ - union v_atomic last_tsc; /* - * Last timestamp written in the buffer. - */ - - struct lib_ring_buffer_backend backend; /* Associated backend */ - - struct commit_counters_cold *commit_cold; - /* Commit count per sub-buffer */ - u64 *ts_end; /* - * timestamp_end per sub-buffer. - * Time is sampled by the - * switch_*_end() callbacks which - * are the last space reservation - * performed in the sub-buffer - * before it can be fully - * committed and delivered. This - * time value is then read by - * the deliver callback, - * performed by the last commit - * before the buffer becomes - * readable. - */ - atomic_long_t active_readers; /* - * Active readers count - * standard atomic access (shared) - */ - /* Dropped records */ - union v_atomic records_lost_full; /* Buffer full */ - union v_atomic records_lost_wrap; /* Nested wrap-around */ - union v_atomic records_lost_big; /* Events too big */ - union v_atomic records_count; /* Number of records written */ - union v_atomic records_overrun; /* Number of overwritten records */ - wait_queue_head_t read_wait; /* reader buffer-level wait queue */ - wait_queue_head_t write_wait; /* writer buffer-level wait queue (for metadata only) */ - int finalized; /* buffer has been finalized */ - struct timer_list switch_timer; /* timer for periodical switch */ - struct timer_list read_timer; /* timer for read poll */ - raw_spinlock_t raw_tick_nohz_spinlock; /* nohz entry lock/trylock */ - struct lib_ring_buffer_iter iter; /* read-side iterator */ - unsigned long get_subbuf_consumed; /* Read-side consumed */ - unsigned long prod_snapshot; /* Producer count snapshot */ - unsigned long cons_snapshot; /* Consumer count snapshot */ - unsigned int get_subbuf:1, /* Sub-buffer being held by reader */ - switch_timer_enabled:1, /* Protected by ring_buffer_nohz_lock */ - read_timer_enabled:1, /* Protected by ring_buffer_nohz_lock */ - quiescent:1; -}; - -static inline -void *channel_get_private(struct channel *chan) -{ - return chan->backend.priv; -} - -void lib_ring_buffer_lost_event_too_big(struct channel *chan); - -/* - * Issue warnings and disable channels upon internal error. - * Can receive struct lib_ring_buffer or struct lib_ring_buffer_backend - * parameters. - */ -#define CHAN_WARN_ON(c, cond) \ - ({ \ - struct channel *__chan; \ - int _____ret = unlikely(cond); \ - if (_____ret) { \ - if (__same_type(*(c), struct channel_backend)) \ - __chan = container_of((void *) (c), \ - struct channel, \ - backend); \ - else if (__same_type(*(c), struct channel)) \ - __chan = (void *) (c); \ - else \ - BUG_ON(1); \ - atomic_inc(&__chan->record_disabled); \ - WARN_ON(1); \ - } \ - _____ret; \ - }) - -#endif /* _LIB_RING_BUFFER_FRONTEND_TYPES_H */ diff --git a/lib/ringbuffer/iterator.h b/lib/ringbuffer/iterator.h deleted file mode 100644 index 8064e42a..00000000 --- a/lib/ringbuffer/iterator.h +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/iterator.h - * - * Ring buffer and channel iterators. - * - * Copyright (C) 2010-2012 Mathieu Desnoyers - */ - -#ifndef _LIB_RING_BUFFER_ITERATOR_H -#define _LIB_RING_BUFFER_ITERATOR_H - -#include -#include -#include - -/* - * lib_ring_buffer_get_next_record advances the buffer read position to the next - * record. It returns either the size of the next record, -EAGAIN if there is - * currently no data available, or -ENODATA if no data is available and buffer - * is finalized. - */ -extern ssize_t lib_ring_buffer_get_next_record(struct channel *chan, - struct lib_ring_buffer *buf); - -/* - * channel_get_next_record advances the buffer read position to the next record. - * It returns either the size of the next record, -EAGAIN if there is currently - * no data available, or -ENODATA if no data is available and buffer is - * finalized. - * Returns the current buffer in ret_buf. - */ -extern ssize_t channel_get_next_record(struct channel *chan, - struct lib_ring_buffer **ret_buf); - -/** - * read_current_record - copy the buffer current record into dest. - * @buf: ring buffer - * @dest: destination where the record should be copied - * - * dest should be large enough to contain the record. Returns the number of - * bytes copied. - */ -static inline size_t read_current_record(struct lib_ring_buffer *buf, void *dest) -{ - return lib_ring_buffer_read(&buf->backend, buf->iter.read_offset, - dest, buf->iter.payload_len); -} - -extern int lib_ring_buffer_iterator_open(struct lib_ring_buffer *buf); -extern void lib_ring_buffer_iterator_release(struct lib_ring_buffer *buf); -extern int channel_iterator_open(struct channel *chan); -extern void channel_iterator_release(struct channel *chan); - -extern const struct file_operations channel_payload_file_operations; -extern const struct file_operations lib_ring_buffer_payload_file_operations; - -/* - * Used internally. - */ -int channel_iterator_init(struct channel *chan); -void channel_iterator_unregister_notifiers(struct channel *chan); -void channel_iterator_free(struct channel *chan); -void channel_iterator_reset(struct channel *chan); -void lib_ring_buffer_iterator_reset(struct lib_ring_buffer *buf); - -#endif /* _LIB_RING_BUFFER_ITERATOR_H */ diff --git a/lib/ringbuffer/nohz.h b/lib/ringbuffer/nohz.h deleted file mode 100644 index 1d28d279..00000000 --- a/lib/ringbuffer/nohz.h +++ /dev/null @@ -1,29 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/nohz.h - * - * Copyright (C) 2011-2012 Mathieu Desnoyers - */ - -#ifndef _LIB_RING_BUFFER_NOHZ_H -#define _LIB_RING_BUFFER_NOHZ_H - -#ifdef CONFIG_LIB_RING_BUFFER -void lib_ring_buffer_tick_nohz_flush(void); -void lib_ring_buffer_tick_nohz_stop(void); -void lib_ring_buffer_tick_nohz_restart(void); -#else -static inline void lib_ring_buffer_tick_nohz_flush(void) -{ -} - -static inline void lib_ring_buffer_tick_nohz_stop(void) -{ -} - -static inline void lib_ring_buffer_tick_nohz_restart(void) -{ -} -#endif - -#endif /* _LIB_RING_BUFFER_NOHZ_H */ diff --git a/lib/ringbuffer/ring_buffer_backend.c b/lib/ringbuffer/ring_buffer_backend.c index ec331232..ec819703 100644 --- a/lib/ringbuffer/ring_buffer_backend.c +++ b/lib/ringbuffer/ring_buffer_backend.c @@ -17,9 +17,9 @@ #include #include -#include -#include -#include +#include +#include +#include /** * lib_ring_buffer_backend_allocate - allocate a channel buffer diff --git a/lib/ringbuffer/ring_buffer_frontend.c b/lib/ringbuffer/ring_buffer_frontend.c index 8ffb702e..28f19cd4 100644 --- a/lib/ringbuffer/ring_buffer_frontend.c +++ b/lib/ringbuffer/ring_buffer_frontend.c @@ -45,11 +45,11 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include /* * Internal structure representing offsets to use at a sub-buffer switch. diff --git a/lib/ringbuffer/ring_buffer_iterator.c b/lib/ringbuffer/ring_buffer_iterator.c index 96f4c3b2..eecf24fb 100644 --- a/lib/ringbuffer/ring_buffer_iterator.c +++ b/lib/ringbuffer/ring_buffer_iterator.c @@ -9,7 +9,7 @@ * Copyright (C) 2010-2012 Mathieu Desnoyers */ -#include +#include #include #include #include diff --git a/lib/ringbuffer/ring_buffer_mmap.c b/lib/ringbuffer/ring_buffer_mmap.c index def29b18..ad672888 100644 --- a/lib/ringbuffer/ring_buffer_mmap.c +++ b/lib/ringbuffer/ring_buffer_mmap.c @@ -13,9 +13,9 @@ #include #include -#include -#include -#include +#include +#include +#include /* * fault() vm_op implementation for ring buffer file mapping. diff --git a/lib/ringbuffer/ring_buffer_splice.c b/lib/ringbuffer/ring_buffer_splice.c index 581e2ef3..db1f8867 100644 --- a/lib/ringbuffer/ring_buffer_splice.c +++ b/lib/ringbuffer/ring_buffer_splice.c @@ -14,9 +14,9 @@ #include #include -#include -#include -#include +#include +#include +#include #if 0 #define printk_dbg(fmt, args...) printk(fmt, args) diff --git a/lib/ringbuffer/ring_buffer_vfs.c b/lib/ringbuffer/ring_buffer_vfs.c index 99000613..896543e3 100644 --- a/lib/ringbuffer/ring_buffer_vfs.c +++ b/lib/ringbuffer/ring_buffer_vfs.c @@ -11,10 +11,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include static int put_ulong(unsigned long val, unsigned long arg) { diff --git a/lib/ringbuffer/vatomic.h b/lib/ringbuffer/vatomic.h deleted file mode 100644 index 6fdc4d1b..00000000 --- a/lib/ringbuffer/vatomic.h +++ /dev/null @@ -1,84 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/vatomic.h - * - * Copyright (C) 2010-2012 Mathieu Desnoyers - */ - -#ifndef _LIB_RING_BUFFER_VATOMIC_H -#define _LIB_RING_BUFFER_VATOMIC_H - -#include -#include - -/* - * Same data type (long) accessed differently depending on configuration. - * v field is for non-atomic access (protected by mutual exclusion). - * In the fast-path, the ring_buffer_config structure is constant, so the - * compiler can statically select the appropriate branch. - * local_t is used for per-cpu and per-thread buffers. - * atomic_long_t is used for globally shared buffers. - */ -union v_atomic { - local_t l; - atomic_long_t a; - long v; -}; - -static inline -long v_read(const struct lib_ring_buffer_config *config, union v_atomic *v_a) -{ - if (config->sync == RING_BUFFER_SYNC_PER_CPU) - return local_read(&v_a->l); - else - return atomic_long_read(&v_a->a); -} - -static inline -void v_set(const struct lib_ring_buffer_config *config, union v_atomic *v_a, - long v) -{ - if (config->sync == RING_BUFFER_SYNC_PER_CPU) - local_set(&v_a->l, v); - else - atomic_long_set(&v_a->a, v); -} - -static inline -void v_add(const struct lib_ring_buffer_config *config, long v, union v_atomic *v_a) -{ - if (config->sync == RING_BUFFER_SYNC_PER_CPU) - local_add(v, &v_a->l); - else - atomic_long_add(v, &v_a->a); -} - -static inline -void v_inc(const struct lib_ring_buffer_config *config, union v_atomic *v_a) -{ - if (config->sync == RING_BUFFER_SYNC_PER_CPU) - local_inc(&v_a->l); - else - atomic_long_inc(&v_a->a); -} - -/* - * Non-atomic decrement. Only used by reader, apply to reader-owned subbuffer. - */ -static inline -void _v_dec(const struct lib_ring_buffer_config *config, union v_atomic *v_a) -{ - --v_a->v; -} - -static inline -long v_cmpxchg(const struct lib_ring_buffer_config *config, union v_atomic *v_a, - long old, long _new) -{ - if (config->sync == RING_BUFFER_SYNC_PER_CPU) - return local_cmpxchg(&v_a->l, old, _new); - else - return atomic_long_cmpxchg(&v_a->a, old, _new); -} - -#endif /* _LIB_RING_BUFFER_VATOMIC_H */ diff --git a/lib/ringbuffer/vfs.h b/lib/ringbuffer/vfs.h deleted file mode 100644 index ee23a624..00000000 --- a/lib/ringbuffer/vfs.h +++ /dev/null @@ -1,166 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only) - * - * lib/ringbuffer/vfs.h - * - * Wait-free ring buffer VFS file operations. - * - * Copyright (C) 2005-2012 Mathieu Desnoyers - */ - -#ifndef _LIB_RING_BUFFER_VFS_H -#define _LIB_RING_BUFFER_VFS_H - -#include -#include - -/* VFS API */ - -extern const struct file_operations lib_ring_buffer_file_operations; - -/* - * Internal file operations. - */ - -struct lib_ring_buffer; - -int lib_ring_buffer_open(struct inode *inode, struct file *file, - struct lib_ring_buffer *buf); -int lib_ring_buffer_release(struct inode *inode, struct file *file, - struct lib_ring_buffer *buf); -unsigned int lib_ring_buffer_poll(struct file *filp, poll_table *wait, - struct lib_ring_buffer *buf); -ssize_t lib_ring_buffer_splice_read(struct file *in, loff_t *ppos, - struct pipe_inode_info *pipe, size_t len, - unsigned int flags, struct lib_ring_buffer *buf); -int lib_ring_buffer_mmap(struct file *filp, struct vm_area_struct *vma, - struct lib_ring_buffer *buf); - -/* Ring Buffer ioctl() and ioctl numbers */ -long lib_ring_buffer_ioctl(struct file *filp, unsigned int cmd, - unsigned long arg, struct lib_ring_buffer *buf); -#ifdef CONFIG_COMPAT -long lib_ring_buffer_compat_ioctl(struct file *filp, unsigned int cmd, - unsigned long arg, struct lib_ring_buffer *buf); -#endif - -ssize_t vfs_lib_ring_buffer_file_splice_read(struct file *in, loff_t *ppos, - struct pipe_inode_info *pipe, size_t len, unsigned int flags); -loff_t vfs_lib_ring_buffer_no_llseek(struct file *file, loff_t offset, - int origin); -int vfs_lib_ring_buffer_mmap(struct file *filp, struct vm_area_struct *vma); -ssize_t vfs_lib_ring_buffer_splice_read(struct file *in, loff_t *ppos, - struct pipe_inode_info *pipe, size_t len, - unsigned int flags); - -/* - * Use RING_BUFFER_GET_NEXT_SUBBUF / RING_BUFFER_PUT_NEXT_SUBBUF to read and - * consume sub-buffers sequentially. - * - * Reading sub-buffers without consuming them can be performed with: - * - * RING_BUFFER_SNAPSHOT - * RING_BUFFER_SNAPSHOT_GET_CONSUMED - * RING_BUFFER_SNAPSHOT_GET_PRODUCED - * - * to get the offset range to consume, and then by passing each sub-buffer - * offset to RING_BUFFER_GET_SUBBUF, read the sub-buffer, and then release it - * with RING_BUFFER_PUT_SUBBUF. - * - * Note that the "snapshot" API can be used to read the sub-buffer in reverse - * order, which is useful for flight recorder snapshots. - */ - -/* Get a snapshot of the current ring buffer producer and consumer positions */ -#define RING_BUFFER_SNAPSHOT _IO(0xF6, 0x00) -/* Get the consumer position (iteration start) */ -#define RING_BUFFER_SNAPSHOT_GET_CONSUMED _IOR(0xF6, 0x01, unsigned long) -/* Get the producer position (iteration end) */ -#define RING_BUFFER_SNAPSHOT_GET_PRODUCED _IOR(0xF6, 0x02, unsigned long) -/* Get exclusive read access to the specified sub-buffer position */ -#define RING_BUFFER_GET_SUBBUF _IOW(0xF6, 0x03, unsigned long) -/* Release exclusive sub-buffer access */ -#define RING_BUFFER_PUT_SUBBUF _IO(0xF6, 0x04) - -/* Get exclusive read access to the next sub-buffer that can be read. */ -#define RING_BUFFER_GET_NEXT_SUBBUF _IO(0xF6, 0x05) -/* Release exclusive sub-buffer access, move consumer forward. */ -#define RING_BUFFER_PUT_NEXT_SUBBUF _IO(0xF6, 0x06) -/* returns the size of the current sub-buffer, without padding (for mmap). */ -#define RING_BUFFER_GET_SUBBUF_SIZE _IOR(0xF6, 0x07, unsigned long) -/* returns the size of the current sub-buffer, with padding (for splice). */ -#define RING_BUFFER_GET_PADDED_SUBBUF_SIZE _IOR(0xF6, 0x08, unsigned long) -/* returns the maximum size for sub-buffers. */ -#define RING_BUFFER_GET_MAX_SUBBUF_SIZE _IOR(0xF6, 0x09, unsigned long) -/* returns the length to mmap. */ -#define RING_BUFFER_GET_MMAP_LEN _IOR(0xF6, 0x0A, unsigned long) -/* returns the offset of the subbuffer belonging to the mmap reader. */ -#define RING_BUFFER_GET_MMAP_READ_OFFSET _IOR(0xF6, 0x0B, unsigned long) -/* Flush the current sub-buffer, if non-empty. */ -#define RING_BUFFER_FLUSH _IO(0xF6, 0x0C) -/* Get the current version of the metadata cache (after a get_next). */ -#define RING_BUFFER_GET_METADATA_VERSION _IOR(0xF6, 0x0D, uint64_t) -/* - * Get a snapshot of the current ring buffer producer and consumer positions, - * regardless of whether or not the two positions are contained within the same - * sub-buffer. - */ -#define RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS _IO(0xF6, 0x0E) -/* Flush the current sub-buffer, even if empty. */ -#define RING_BUFFER_FLUSH_EMPTY _IO(0xF6, 0x0F) -/* - * Reset the position of what has been consumed from the metadata cache to 0 - * so it can be read again. - */ -#define RING_BUFFER_METADATA_CACHE_DUMP _IO(0xF6, 0x10) -/* Clear ring buffer content. */ -#define RING_BUFFER_CLEAR _IO(0xF6, 0x11) - -#ifdef CONFIG_COMPAT -/* Get a snapshot of the current ring buffer producer and consumer positions */ -#define RING_BUFFER_COMPAT_SNAPSHOT RING_BUFFER_SNAPSHOT -/* Get the consumer position (iteration start) */ -#define RING_BUFFER_COMPAT_SNAPSHOT_GET_CONSUMED \ - _IOR(0xF6, 0x01, compat_ulong_t) -/* Get the producer position (iteration end) */ -#define RING_BUFFER_COMPAT_SNAPSHOT_GET_PRODUCED \ - _IOR(0xF6, 0x02, compat_ulong_t) -/* Get exclusive read access to the specified sub-buffer position */ -#define RING_BUFFER_COMPAT_GET_SUBBUF _IOW(0xF6, 0x03, compat_ulong_t) -/* Release exclusive sub-buffer access */ -#define RING_BUFFER_COMPAT_PUT_SUBBUF RING_BUFFER_PUT_SUBBUF - -/* Get exclusive read access to the next sub-buffer that can be read. */ -#define RING_BUFFER_COMPAT_GET_NEXT_SUBBUF RING_BUFFER_GET_NEXT_SUBBUF -/* Release exclusive sub-buffer access, move consumer forward. */ -#define RING_BUFFER_COMPAT_PUT_NEXT_SUBBUF RING_BUFFER_PUT_NEXT_SUBBUF -/* returns the size of the current sub-buffer, without padding (for mmap). */ -#define RING_BUFFER_COMPAT_GET_SUBBUF_SIZE _IOR(0xF6, 0x07, compat_ulong_t) -/* returns the size of the current sub-buffer, with padding (for splice). */ -#define RING_BUFFER_COMPAT_GET_PADDED_SUBBUF_SIZE \ - _IOR(0xF6, 0x08, compat_ulong_t) -/* returns the maximum size for sub-buffers. */ -#define RING_BUFFER_COMPAT_GET_MAX_SUBBUF_SIZE _IOR(0xF6, 0x09, compat_ulong_t) -/* returns the length to mmap. */ -#define RING_BUFFER_COMPAT_GET_MMAP_LEN _IOR(0xF6, 0x0A, compat_ulong_t) -/* returns the offset of the subbuffer belonging to the mmap reader. */ -#define RING_BUFFER_COMPAT_GET_MMAP_READ_OFFSET _IOR(0xF6, 0x0B, compat_ulong_t) -/* Flush the current sub-buffer, if non-empty. */ -#define RING_BUFFER_COMPAT_FLUSH RING_BUFFER_FLUSH -/* Get the current version of the metadata cache (after a get_next). */ -#define RING_BUFFER_COMPAT_GET_METADATA_VERSION RING_BUFFER_GET_METADATA_VERSION -/* - * Get a snapshot of the current ring buffer producer and consumer positions, - * regardless of whether or not the two positions are contained within the same - * sub-buffer. - */ -#define RING_BUFFER_COMPAT_SNAPSHOT_SAMPLE_POSITIONS \ - RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS -/* Flush the current sub-buffer, even if empty. */ -#define RING_BUFFER_COMPAT_FLUSH_EMPTY \ - RING_BUFFER_FLUSH_EMPTY -/* Clear ring buffer content. */ -#define RING_BUFFER_COMPAT_CLEAR \ - RING_BUFFER_CLEAR -#endif /* CONFIG_COMPAT */ - -#endif /* _LIB_RING_BUFFER_VFS_H */ diff --git a/lttng-abi.c b/lttng-abi.c index a32f419a..386c4e31 100644 --- a/lttng-abi.c +++ b/lttng-abi.c @@ -31,16 +31,16 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* * This is LTTng's own personal way to create a system call as an external diff --git a/lttng-calibrate.c b/lttng-calibrate.c index ef6c6121..d137f801 100644 --- a/lttng-calibrate.c +++ b/lttng-calibrate.c @@ -7,8 +7,8 @@ * Copyright (C) 2010-2012 Mathieu Desnoyers */ -#include -#include +#include +#include noinline void lttng_calibrate_kretprobe(void) diff --git a/lttng-clock.c b/lttng-clock.c index 1fe37b80..0d11e839 100644 --- a/lttng-clock.c +++ b/lttng-clock.c @@ -10,8 +10,8 @@ #include #include -#include -#include +#include +#include struct lttng_trace_clock *lttng_trace_clock; EXPORT_SYMBOL_GPL(lttng_trace_clock); diff --git a/lttng-context-callstack.c b/lttng-context-callstack.c index ec7bd62a..e747e19a 100644 --- a/lttng-context-callstack.c +++ b/lttng-context-callstack.c @@ -43,11 +43,11 @@ #include #include #include -#include "lttng-events.h" -#include "include/ringbuffer/backend.h" -#include "include/ringbuffer/frontend.h" -#include "lttng-tracer.h" -#include "lttng-endian.h" +#include +#include +#include +#include +#include #ifdef CONFIG_ARCH_STACKWALK #include "lttng-context-callstack-stackwalk-impl.h" diff --git a/lttng-context-cgroup-ns.c b/lttng-context-cgroup-ns.c index 1c0fc436..b2f20c4f 100644 --- a/lttng-context-cgroup-ns.c +++ b/lttng-context-cgroup-ns.c @@ -13,9 +13,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #if defined(CONFIG_CGROUPS) diff --git a/lttng-context-cpu-id.c b/lttng-context-cpu-id.c index dbd01234..ef4f912b 100644 --- a/lttng-context-cpu-id.c +++ b/lttng-context-cpu-id.c @@ -10,9 +10,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t cpu_id_get_size(size_t offset) diff --git a/lttng-context-egid.c b/lttng-context-egid.c index 4921ff4a..e4b70816 100644 --- a/lttng-context-egid.c +++ b/lttng-context-egid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-euid.c b/lttng-context-euid.c index 48df6e1c..fb13d027 100644 --- a/lttng-context-euid.c +++ b/lttng-context-euid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-gid.c b/lttng-context-gid.c index 58859c23..d573082e 100644 --- a/lttng-context-gid.c +++ b/lttng-context-gid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-hostname.c b/lttng-context-hostname.c index 5f31d513..94cb7e5e 100644 --- a/lttng-context-hostname.c +++ b/lttng-context-hostname.c @@ -11,9 +11,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #define LTTNG_HOSTNAME_CTX_LEN (__NEW_UTS_LEN + 1) diff --git a/lttng-context-interruptible.c b/lttng-context-interruptible.c index d354a088..3f80450b 100644 --- a/lttng-context-interruptible.c +++ b/lttng-context-interruptible.c @@ -11,9 +11,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include /* * Interruptible at value -1 means "unknown". diff --git a/lttng-context-ipc-ns.c b/lttng-context-ipc-ns.c index e5e7d6ee..13470f50 100644 --- a/lttng-context-ipc-ns.c +++ b/lttng-context-ipc-ns.c @@ -13,9 +13,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #if defined(CONFIG_IPC_NS) diff --git a/lttng-context-migratable.c b/lttng-context-migratable.c index 607d6ef3..68983bcd 100644 --- a/lttng-context-migratable.c +++ b/lttng-context-migratable.c @@ -11,9 +11,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t migratable_get_size(size_t offset) diff --git a/lttng-context-mnt-ns.c b/lttng-context-mnt-ns.c index 2ede85af..7a2e0605 100644 --- a/lttng-context-mnt-ns.c +++ b/lttng-context-mnt-ns.c @@ -12,10 +12,10 @@ #include #include #include -#include +#include #include -#include -#include +#include +#include #if !defined(LTTNG_MNT_NS_MISSING_HEADER) diff --git a/lttng-context-need-reschedule.c b/lttng-context-need-reschedule.c index 9bbb43ed..275b00d0 100644 --- a/lttng-context-need-reschedule.c +++ b/lttng-context-need-reschedule.c @@ -11,9 +11,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t need_reschedule_get_size(size_t offset) diff --git a/lttng-context-net-ns.c b/lttng-context-net-ns.c index 79207301..490ea6f0 100644 --- a/lttng-context-net-ns.c +++ b/lttng-context-net-ns.c @@ -14,9 +14,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #if defined(CONFIG_NET_NS) diff --git a/lttng-context-nice.c b/lttng-context-nice.c index 0c7911c1..58a8d96e 100644 --- a/lttng-context-nice.c +++ b/lttng-context-nice.c @@ -10,9 +10,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t nice_get_size(size_t offset) diff --git a/lttng-context-perf-counters.c b/lttng-context-perf-counters.c index 2365d03f..9fa6db50 100644 --- a/lttng-context-perf-counters.c +++ b/lttng-context-perf-counters.c @@ -14,9 +14,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t perf_counter_get_size(size_t offset) diff --git a/lttng-context-pid-ns.c b/lttng-context-pid-ns.c index b22d4460..ae4def74 100644 --- a/lttng-context-pid-ns.c +++ b/lttng-context-pid-ns.c @@ -13,9 +13,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #if defined(CONFIG_PID_NS) diff --git a/lttng-context-pid.c b/lttng-context-pid.c index 3e88eb82..dc196ee1 100644 --- a/lttng-context-pid.c +++ b/lttng-context-pid.c @@ -10,9 +10,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t pid_get_size(size_t offset) diff --git a/lttng-context-ppid.c b/lttng-context-ppid.c index 78d14c27..799246d7 100644 --- a/lttng-context-ppid.c +++ b/lttng-context-ppid.c @@ -11,9 +11,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t ppid_get_size(size_t offset) diff --git a/lttng-context-preemptible.c b/lttng-context-preemptible.c index 336c582a..72e30eff 100644 --- a/lttng-context-preemptible.c +++ b/lttng-context-preemptible.c @@ -11,9 +11,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include /* * We nest twice in preempt disabling within LTTng: one nesting is done diff --git a/lttng-context-prio.c b/lttng-context-prio.c index e23115ef..6050d475 100644 --- a/lttng-context-prio.c +++ b/lttng-context-prio.c @@ -10,9 +10,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t prio_get_size(size_t offset) diff --git a/lttng-context-procname.c b/lttng-context-procname.c index dfd5ab9b..d950096a 100644 --- a/lttng-context-procname.c +++ b/lttng-context-procname.c @@ -10,10 +10,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include static size_t procname_get_size(size_t offset) diff --git a/lttng-context-sgid.c b/lttng-context-sgid.c index 464dd911..8703f966 100644 --- a/lttng-context-sgid.c +++ b/lttng-context-sgid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-suid.c b/lttng-context-suid.c index a7a1b22e..878eaec0 100644 --- a/lttng-context-suid.c +++ b/lttng-context-suid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-tid.c b/lttng-context-tid.c index b08ffbfe..91ea655b 100644 --- a/lttng-context-tid.c +++ b/lttng-context-tid.c @@ -10,9 +10,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t tid_get_size(size_t offset) diff --git a/lttng-context-uid.c b/lttng-context-uid.c index 43198184..4dd754d4 100644 --- a/lttng-context-uid.c +++ b/lttng-context-uid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-user-ns.c b/lttng-context-user-ns.c index 1dc3f7eb..51cb6b52 100644 --- a/lttng-context-user-ns.c +++ b/lttng-context-user-ns.c @@ -13,9 +13,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #if defined(CONFIG_USER_NS) diff --git a/lttng-context-uts-ns.c b/lttng-context-uts-ns.c index 684c3216..7141f617 100644 --- a/lttng-context-uts-ns.c +++ b/lttng-context-uts-ns.c @@ -13,9 +13,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #if defined(CONFIG_UTS_NS) diff --git a/lttng-context-vegid.c b/lttng-context-vegid.c index fa707a41..d0db2f8c 100644 --- a/lttng-context-vegid.c +++ b/lttng-context-vegid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-veuid.c b/lttng-context-veuid.c index 1db2283d..19d6e1bc 100644 --- a/lttng-context-veuid.c +++ b/lttng-context-veuid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-vgid.c b/lttng-context-vgid.c index 7aacf5e4..ec5f9965 100644 --- a/lttng-context-vgid.c +++ b/lttng-context-vgid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-vpid.c b/lttng-context-vpid.c index eb0bcbd9..8d9b33c1 100644 --- a/lttng-context-vpid.c +++ b/lttng-context-vpid.c @@ -10,9 +10,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t vpid_get_size(size_t offset) diff --git a/lttng-context-vppid.c b/lttng-context-vppid.c index c370d372..c379b726 100644 --- a/lttng-context-vppid.c +++ b/lttng-context-vppid.c @@ -11,9 +11,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t vppid_get_size(size_t offset) diff --git a/lttng-context-vsgid.c b/lttng-context-vsgid.c index c98a03a5..a2256249 100644 --- a/lttng-context-vsgid.c +++ b/lttng-context-vsgid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-vsuid.c b/lttng-context-vsuid.c index e85a9170..1299b3d0 100644 --- a/lttng-context-vsuid.c +++ b/lttng-context-vsuid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context-vtid.c b/lttng-context-vtid.c index 7f482b07..6fd6bfc5 100644 --- a/lttng-context-vtid.c +++ b/lttng-context-vtid.c @@ -10,9 +10,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include static size_t vtid_get_size(size_t offset) diff --git a/lttng-context-vuid.c b/lttng-context-vuid.c index 95360521..cd20606a 100644 --- a/lttng-context-vuid.c +++ b/lttng-context-vuid.c @@ -11,9 +11,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include static diff --git a/lttng-context.c b/lttng-context.c index 9f841727..7c8f5d92 100644 --- a/lttng-context.c +++ b/lttng-context.c @@ -11,8 +11,8 @@ #include #include #include -#include -#include +#include +#include /* * The filter implementation requires that two consecutive "get" for the diff --git a/lttng-events.c b/lttng-events.c index 5ecdd732..fb45c612 100644 --- a/lttng-events.c +++ b/lttng-events.c @@ -25,15 +25,15 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #define METADATA_CACHE_DEFAULT_SIZE 4096 diff --git a/lttng-filter-interpreter.c b/lttng-filter-interpreter.c index 671f8b8a..15f4747d 100644 --- a/lttng-filter-interpreter.c +++ b/lttng-filter-interpreter.c @@ -12,8 +12,8 @@ #include #include -#include -#include +#include +#include STACK_FRAME_NON_STANDARD(lttng_filter_interpret_bytecode); diff --git a/lttng-filter-specialize.c b/lttng-filter-specialize.c index 3dc82d84..02aef150 100644 --- a/lttng-filter-specialize.c +++ b/lttng-filter-specialize.c @@ -8,8 +8,8 @@ */ #include -#include -#include "lib/align.h" +#include +#include static ssize_t bytecode_reserve_data(struct bytecode_runtime *runtime, size_t align, size_t len) diff --git a/lttng-filter-validator.c b/lttng-filter-validator.c index c9094547..e27f6b91 100644 --- a/lttng-filter-validator.c +++ b/lttng-filter-validator.c @@ -11,7 +11,7 @@ #include #include -#include +#include #define MERGE_POINT_TABLE_BITS 7 #define MERGE_POINT_TABLE_SIZE (1U << MERGE_POINT_TABLE_BITS) diff --git a/lttng-filter.c b/lttng-filter.c index 45319d6f..71479336 100644 --- a/lttng-filter.c +++ b/lttng-filter.c @@ -10,7 +10,7 @@ #include #include -#include +#include static const char *opnames[] = { [ FILTER_OP_UNKNOWN ] = "UNKNOWN", diff --git a/lttng-probes.c b/lttng-probes.c index bf6615b7..c6ab3fcd 100644 --- a/lttng-probes.c +++ b/lttng-probes.c @@ -12,7 +12,7 @@ #include #include -#include +#include /* * probe list is protected by sessions lock. diff --git a/lttng-ring-buffer-client-discard.c b/lttng-ring-buffer-client-discard.c index 64d7e9ec..d8553259 100644 --- a/lttng-ring-buffer-client-discard.c +++ b/lttng-ring-buffer-client-discard.c @@ -8,7 +8,7 @@ */ #include -#include +#include #define RING_BUFFER_MODE_TEMPLATE RING_BUFFER_DISCARD #define RING_BUFFER_MODE_TEMPLATE_STRING "discard" diff --git a/lttng-ring-buffer-client-mmap-discard.c b/lttng-ring-buffer-client-mmap-discard.c index 9147e58b..4a0b5628 100644 --- a/lttng-ring-buffer-client-mmap-discard.c +++ b/lttng-ring-buffer-client-mmap-discard.c @@ -8,7 +8,7 @@ */ #include -#include +#include #define RING_BUFFER_MODE_TEMPLATE RING_BUFFER_DISCARD #define RING_BUFFER_MODE_TEMPLATE_STRING "discard-mmap" diff --git a/lttng-ring-buffer-client-mmap-overwrite.c b/lttng-ring-buffer-client-mmap-overwrite.c index 4494ac4b..293328dc 100644 --- a/lttng-ring-buffer-client-mmap-overwrite.c +++ b/lttng-ring-buffer-client-mmap-overwrite.c @@ -8,7 +8,7 @@ */ #include -#include +#include #define RING_BUFFER_MODE_TEMPLATE RING_BUFFER_OVERWRITE #define RING_BUFFER_MODE_TEMPLATE_STRING "overwrite-mmap" diff --git a/lttng-ring-buffer-client-overwrite.c b/lttng-ring-buffer-client-overwrite.c index d4566296..973573a6 100644 --- a/lttng-ring-buffer-client-overwrite.c +++ b/lttng-ring-buffer-client-overwrite.c @@ -8,7 +8,7 @@ */ #include -#include +#include #define RING_BUFFER_MODE_TEMPLATE RING_BUFFER_OVERWRITE #define RING_BUFFER_MODE_TEMPLATE_STRING "overwrite" diff --git a/lttng-ring-buffer-client.h b/lttng-ring-buffer-client.h index ffbb1f46..94d3a64a 100644 --- a/lttng-ring-buffer-client.h +++ b/lttng-ring-buffer-client.h @@ -9,11 +9,11 @@ #include #include -#include +#include #include -#include -#include -#include +#include +#include +#include #define LTTNG_COMPACT_EVENT_BITS 5 #define LTTNG_COMPACT_TSC_BITS 27 @@ -190,7 +190,7 @@ size_t record_header_size(const struct lib_ring_buffer_config *config, return offset - orig_offset; } -#include +#include static void lttng_write_event_header_slow(const struct lib_ring_buffer_config *config, diff --git a/lttng-ring-buffer-metadata-client.c b/lttng-ring-buffer-metadata-client.c index 7a785677..6032a903 100644 --- a/lttng-ring-buffer-metadata-client.c +++ b/lttng-ring-buffer-metadata-client.c @@ -8,7 +8,7 @@ */ #include -#include +#include #define RING_BUFFER_MODE_TEMPLATE RING_BUFFER_DISCARD #define RING_BUFFER_MODE_TEMPLATE_STRING "metadata" diff --git a/lttng-ring-buffer-metadata-client.h b/lttng-ring-buffer-metadata-client.h index 50cd3779..3dbf3f0a 100644 --- a/lttng-ring-buffer-metadata-client.h +++ b/lttng-ring-buffer-metadata-client.h @@ -9,8 +9,8 @@ #include #include -#include -#include +#include +#include static struct lttng_transport lttng_relay_transport; @@ -50,7 +50,7 @@ size_t record_header_size(const struct lib_ring_buffer_config *config, return 0; } -#include +#include static u64 client_ring_buffer_clock_read(struct channel *chan) { diff --git a/lttng-ring-buffer-metadata-mmap-client.c b/lttng-ring-buffer-metadata-mmap-client.c index fc33f99b..6fc6156d 100644 --- a/lttng-ring-buffer-metadata-mmap-client.c +++ b/lttng-ring-buffer-metadata-mmap-client.c @@ -8,7 +8,7 @@ */ #include -#include +#include #define RING_BUFFER_MODE_TEMPLATE RING_BUFFER_DISCARD #define RING_BUFFER_MODE_TEMPLATE_STRING "metadata-mmap" diff --git a/lttng-statedump-impl.c b/lttng-statedump-impl.c index 4635feb0..3d010ca6 100644 --- a/lttng-statedump-impl.c +++ b/lttng-statedump-impl.c @@ -36,8 +36,8 @@ #include #include -#include -#include +#include +#include /* Define the tracepoints, but do not build the probes */ #define CREATE_TRACE_POINTS diff --git a/lttng-string-utils.c b/lttng-string-utils.c index 1f126185..53457a8a 100644 --- a/lttng-string-utils.c +++ b/lttng-string-utils.c @@ -5,7 +5,7 @@ #include -#include +#include enum star_glob_pattern_type_flags { STAR_GLOB_PATTERN_TYPE_FLAG_NONE = 0, diff --git a/lttng-syscalls.c b/lttng-syscalls.c index 41fc2e3d..5fcbe6fc 100644 --- a/lttng-syscalls.c +++ b/lttng-syscalls.c @@ -22,9 +22,9 @@ #include #include -#include -#include -#include "lttng-tracepoint.h" +#include +#include +#include #define LTTNG_SYSCALL_NR_ARGS 6 diff --git a/lttng-tp-mempool.c b/lttng-tp-mempool.c index 7e1b51d3..27919891 100644 --- a/lttng-tp-mempool.c +++ b/lttng-tp-mempool.c @@ -8,7 +8,7 @@ #include #include -#include +#include struct lttng_tp_buf_entry { int cpu; /* To make sure we return the entry to the right pool. */ diff --git a/lttng-tracepoint.c b/lttng-tracepoint.c index 3d0cafd7..797a5ac2 100644 --- a/lttng-tracepoint.c +++ b/lttng-tracepoint.c @@ -15,7 +15,7 @@ #include #include -#include +#include /* * Protect the tracepoint table. lttng_tracepoint_mutex nests within diff --git a/lttng-tracker-id.c b/lttng-tracker-id.c index a789e7c4..fac0ab70 100644 --- a/lttng-tracker-id.c +++ b/lttng-tracker-id.c @@ -15,7 +15,7 @@ #include #include -#include +#include /* * Hash table is allocated and freed when there are no possible diff --git a/probes/Kbuild b/probes/Kbuild index 9f0c1a6d..e3bb7a53 100644 --- a/probes/Kbuild +++ b/probes/Kbuild @@ -4,7 +4,7 @@ TOP_LTTNG_MODULES_DIR := $(shell dirname $(lastword $(MAKEFILE_LIST)))/.. include $(TOP_LTTNG_MODULES_DIR)/Kbuild.common -ccflags-y += -I$(TOP_LTTNG_MODULES_DIR) +ccflags-y += -I$(TOP_LTTNG_MODULES_DIR)/include obj-$(CONFIG_LTTNG) += lttng-probe-sched.o obj-$(CONFIG_LTTNG) += lttng-probe-irq.o diff --git a/probes/lttng-kprobes.c b/probes/lttng-kprobes.c index 650db1fa..a6e96d7a 100644 --- a/probes/lttng-kprobes.c +++ b/probes/lttng-kprobes.c @@ -10,10 +10,10 @@ #include #include #include -#include -#include +#include +#include #include -#include +#include static int lttng_kprobes_handler_pre(struct kprobe *p, struct pt_regs *regs) diff --git a/probes/lttng-kretprobes.c b/probes/lttng-kretprobes.c index 83c26ab1..72b3a971 100644 --- a/probes/lttng-kretprobes.c +++ b/probes/lttng-kretprobes.c @@ -11,10 +11,10 @@ #include #include #include -#include -#include +#include +#include #include -#include +#include enum lttng_kretprobe_type { EVENT_ENTRY = 0, diff --git a/probes/lttng-probe-9p.c b/probes/lttng-probe-9p.c index 7d464104..8bd5c0e2 100644 --- a/probes/lttng-probe-9p.c +++ b/probes/lttng-probe-9p.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-block.c b/probes/lttng-probe-block.c index f6e9aa94..730607e4 100644 --- a/probes/lttng-probe-block.c +++ b/probes/lttng-probe-block.c @@ -9,8 +9,8 @@ #include #include -#include -#include +#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -18,7 +18,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-btrfs.c b/probes/lttng-probe-btrfs.c index 54bce99d..93cb740a 100644 --- a/probes/lttng-probe-btrfs.c +++ b/probes/lttng-probe-btrfs.c @@ -15,7 +15,7 @@ #include <../fs/btrfs/volumes.h> #include <../fs/btrfs/block-group.h> #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-compaction.c b/probes/lttng-probe-compaction.c index b7d5052e..7d229dbe 100644 --- a/probes/lttng-probe-compaction.c +++ b/probes/lttng-probe-compaction.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-ext4.c b/probes/lttng-probe-ext4.c index a8f1795b..31c1ad3e 100644 --- a/probes/lttng-probe-ext4.c +++ b/probes/lttng-probe-ext4.c @@ -14,7 +14,7 @@ #include <../fs/ext4/mballoc.h> #include <../fs/ext4/ext4_extents.h> #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -22,8 +22,8 @@ */ #include -#include -#include "lttng-tracepoint.h" +#include +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-gpio.c b/probes/lttng-probe-gpio.c index 0b90c864..ec33e3db 100644 --- a/probes/lttng-probe-gpio.c +++ b/probes/lttng-probe-gpio.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-i2c.c b/probes/lttng-probe-i2c.c index 2867d173..29a49555 100644 --- a/probes/lttng-probe-i2c.c +++ b/probes/lttng-probe-i2c.c @@ -10,7 +10,7 @@ #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-irq.c b/probes/lttng-probe-irq.c index 15b029dd..983e6824 100644 --- a/probes/lttng-probe-irq.c +++ b/probes/lttng-probe-irq.c @@ -9,7 +9,7 @@ #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -17,7 +17,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-jbd2.c b/probes/lttng-probe-jbd2.c index 095043c4..01b1eab9 100644 --- a/probes/lttng-probe-jbd2.c +++ b/probes/lttng-probe-jbd2.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -17,7 +17,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-kmem.c b/probes/lttng-probe-kmem.c index 01f2d9f1..43437afc 100644 --- a/probes/lttng-probe-kmem.c +++ b/probes/lttng-probe-kmem.c @@ -10,7 +10,7 @@ #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -18,7 +18,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-kvm-x86-mmu.c b/probes/lttng-probe-kvm-x86-mmu.c index d731708b..9efd105c 100644 --- a/probes/lttng-probe-kvm-x86-mmu.c +++ b/probes/lttng-probe-kvm-x86-mmu.c @@ -9,7 +9,7 @@ #include #include -#include +#include #include @@ -17,7 +17,7 @@ * Create the tracepoint static inlines from the kernel to validate that our * trace event macros match the kernel we run on. */ -#include "lttng-tracepoint.h" +#include #include <../../arch/x86/kvm/mmutrace.h> diff --git a/probes/lttng-probe-kvm-x86.c b/probes/lttng-probe-kvm-x86.c index 2eb10ae7..0317c095 100644 --- a/probes/lttng-probe-kvm-x86.c +++ b/probes/lttng-probe-kvm-x86.c @@ -9,7 +9,7 @@ #include #include -#include +#include /* TODO: check for header availability in Makefile */ #include <../../arch/x86/kvm/kvm_emulate.h> @@ -22,7 +22,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-kvm.c b/probes/lttng-probe-kvm.c index 4c6d04c1..796bd8fb 100644 --- a/probes/lttng-probe-kvm.c +++ b/probes/lttng-probe-kvm.c @@ -9,7 +9,7 @@ #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -17,7 +17,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-lock.c b/probes/lttng-probe-lock.c index 3d0a68d6..f9045bc8 100644 --- a/probes/lttng-probe-lock.c +++ b/probes/lttng-probe-lock.c @@ -10,14 +10,14 @@ #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our * trace event macros match the kernel we run on. */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-module.c b/probes/lttng-probe-module.c index e99ee482..cd26cfca 100644 --- a/probes/lttng-probe-module.c +++ b/probes/lttng-probe-module.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -17,7 +17,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-napi.c b/probes/lttng-probe-napi.c index c53003bd..7771f876 100644 --- a/probes/lttng-probe-napi.c +++ b/probes/lttng-probe-napi.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -17,7 +17,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-net.c b/probes/lttng-probe-net.c index c69bc1ab..f1e5a695 100644 --- a/probes/lttng-probe-net.c +++ b/probes/lttng-probe-net.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-power.c b/probes/lttng-probe-power.c index 6133e52f..ba00a367 100644 --- a/probes/lttng-probe-power.c +++ b/probes/lttng-probe-power.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -17,7 +17,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-preemptirq.c b/probes/lttng-probe-preemptirq.c index 73ea22bd..cca9970e 100644 --- a/probes/lttng-probe-preemptirq.c +++ b/probes/lttng-probe-preemptirq.c @@ -10,7 +10,7 @@ #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-printk.c b/probes/lttng-probe-printk.c index 3476bed8..cae8918f 100644 --- a/probes/lttng-probe-printk.c +++ b/probes/lttng-probe-printk.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-random.c b/probes/lttng-probe-random.c index 918b4c20..1353b168 100644 --- a/probes/lttng-probe-random.c +++ b/probes/lttng-probe-random.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-rcu.c b/probes/lttng-probe-rcu.c index df7ddcbf..9b7fa652 100644 --- a/probes/lttng-probe-rcu.c +++ b/probes/lttng-probe-rcu.c @@ -10,7 +10,7 @@ #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-regmap.c b/probes/lttng-probe-regmap.c index 24a03e03..47052cea 100644 --- a/probes/lttng-probe-regmap.c +++ b/probes/lttng-probe-regmap.c @@ -10,8 +10,8 @@ #include #include -#include -#include +#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-regulator.c b/probes/lttng-probe-regulator.c index 51a1e04b..94c97a01 100644 --- a/probes/lttng-probe-regulator.c +++ b/probes/lttng-probe-regulator.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-rpm.c b/probes/lttng-probe-rpm.c index 00ccd91a..c21c3d9a 100644 --- a/probes/lttng-probe-rpm.c +++ b/probes/lttng-probe-rpm.c @@ -10,7 +10,7 @@ #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-sched.c b/probes/lttng-probe-sched.c index 495374d8..854595a3 100644 --- a/probes/lttng-probe-sched.c +++ b/probes/lttng-probe-sched.c @@ -8,7 +8,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -16,7 +16,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-scsi.c b/probes/lttng-probe-scsi.c index 89e43273..786f29a6 100644 --- a/probes/lttng-probe-scsi.c +++ b/probes/lttng-probe-scsi.c @@ -10,7 +10,7 @@ #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-signal.c b/probes/lttng-probe-signal.c index fcbfca13..d6d72ca4 100644 --- a/probes/lttng-probe-signal.c +++ b/probes/lttng-probe-signal.c @@ -8,7 +8,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-skb.c b/probes/lttng-probe-skb.c index ec0469ab..0e0d89ce 100644 --- a/probes/lttng-probe-skb.c +++ b/probes/lttng-probe-skb.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -17,7 +17,7 @@ */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-sock.c b/probes/lttng-probe-sock.c index 1d3a4939..6f90f100 100644 --- a/probes/lttng-probe-sock.c +++ b/probes/lttng-probe-sock.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-statedump.c b/probes/lttng-probe-statedump.c index cade8471..5e93d5ba 100644 --- a/probes/lttng-probe-statedump.c +++ b/probes/lttng-probe-statedump.c @@ -15,8 +15,8 @@ #include #include #include -#include -#include +#include +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-sunrpc.c b/probes/lttng-probe-sunrpc.c index bd911a9c..43d657dd 100644 --- a/probes/lttng-probe-sunrpc.c +++ b/probes/lttng-probe-sunrpc.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-timer.c b/probes/lttng-probe-timer.c index ef036a64..e262db42 100644 --- a/probes/lttng-probe-timer.c +++ b/probes/lttng-probe-timer.c @@ -8,7 +8,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -18,7 +18,7 @@ #include #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-udp.c b/probes/lttng-probe-udp.c index a9fa382b..4968f473 100644 --- a/probes/lttng-probe-udp.c +++ b/probes/lttng-probe-udp.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our diff --git a/probes/lttng-probe-user.c b/probes/lttng-probe-user.c index 411899d1..4f5728ac 100644 --- a/probes/lttng-probe-user.c +++ b/probes/lttng-probe-user.c @@ -7,7 +7,7 @@ #include #include -#include +#include /* * Calculate string length. Include final null terminating character if there is diff --git a/probes/lttng-probe-v4l2.c b/probes/lttng-probe-v4l2.c index 78334477..d14a89db 100644 --- a/probes/lttng-probe-v4l2.c +++ b/probes/lttng-probe-v4l2.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our * trace event macros match the kernel we run on. diff --git a/probes/lttng-probe-vmscan.c b/probes/lttng-probe-vmscan.c index 2cafb965..41204420 100644 --- a/probes/lttng-probe-vmscan.c +++ b/probes/lttng-probe-vmscan.c @@ -9,7 +9,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -17,7 +17,7 @@ */ #include -#include +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-workqueue.c b/probes/lttng-probe-workqueue.c index 66b19f74..15b49cdf 100644 --- a/probes/lttng-probe-workqueue.c +++ b/probes/lttng-probe-workqueue.c @@ -10,7 +10,7 @@ #include #include -#include +#include struct cpu_workqueue_struct; struct pool_workqueue; @@ -21,7 +21,7 @@ struct pool_workqueue; */ #include -#include "lttng-tracepoint.h" +#include /* * Create LTTng tracepoint probes. diff --git a/probes/lttng-probe-writeback.c b/probes/lttng-probe-writeback.c index 14495db9..17f8a0da 100644 --- a/probes/lttng-probe-writeback.c +++ b/probes/lttng-probe-writeback.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -19,7 +19,7 @@ */ #include -#include +#include /* #if */ diff --git a/probes/lttng-probe-x86-exceptions.c b/probes/lttng-probe-x86-exceptions.c index 92603b95..33f16d26 100644 --- a/probes/lttng-probe-x86-exceptions.c +++ b/probes/lttng-probe-x86-exceptions.c @@ -8,7 +8,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -16,7 +16,7 @@ */ #include -#include "lttng-tracepoint.h" +#include #undef TRACE_INCLUDE_PATH #undef TRACE_INCLUDE_FILE diff --git a/probes/lttng-probe-x86-irq-vectors.c b/probes/lttng-probe-x86-irq-vectors.c index a7c8c2a0..52526a44 100644 --- a/probes/lttng-probe-x86-irq-vectors.c +++ b/probes/lttng-probe-x86-irq-vectors.c @@ -8,7 +8,7 @@ */ #include -#include +#include /* * Create the tracepoint static inlines from the kernel to validate that our @@ -16,7 +16,7 @@ */ #include -#include "lttng-tracepoint.h" +#include #undef TRACE_INCLUDE_PATH #undef TRACE_INCLUDE_FILE diff --git a/probes/lttng-uprobes.c b/probes/lttng-uprobes.c index 268b5638..2cc23bb6 100644 --- a/probes/lttng-uprobes.c +++ b/probes/lttng-uprobes.c @@ -16,10 +16,10 @@ #include #include #include -#include -#include +#include +#include #include -#include +#include static int lttng_uprobes_handler_pre(struct uprobe_consumer *uc, struct pt_regs *regs) diff --git a/probes/lttng.c b/probes/lttng.c index ae7dce64..735b167d 100644 --- a/probes/lttng.c +++ b/probes/lttng.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #define TP_MODULE_NOAUTOLOAD #define LTTNG_PACKAGE_BUILD diff --git a/probes/lttng.h b/probes/lttng.h deleted file mode 100644 index 7d153be3..00000000 --- a/probes/lttng.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only) - * - * lttng.h - * - * Copyright (C) 2010-2012 Mathieu Desnoyers - */ - -#ifndef _LTTNG_PROBES_LTTNG_H -#define _LTTNG_PROBES_LTTNG_H - -#undef PARAMS -#define PARAMS(args...) args - -#endif /* _LTTNG_PROBES_LTTNG_H */ diff --git a/tests/Kbuild b/tests/Kbuild index 3e2edf5b..906cbac8 100644 --- a/tests/Kbuild +++ b/tests/Kbuild @@ -4,7 +4,7 @@ TOP_LTTNG_MODULES_DIR := $(shell dirname $(lastword $(MAKEFILE_LIST)))/.. include $(TOP_LTTNG_MODULES_DIR)/Kbuild.common -ccflags-y += -I$(TOP_LTTNG_MODULES_DIR) +ccflags-y += -I$(TOP_LTTNG_MODULES_DIR)/include obj-$(CONFIG_LTTNG) += lttng-test.o lttng-test-objs := probes/lttng-test.o diff --git a/tests/clock-plugin/lttng-clock-plugin-test.c b/tests/clock-plugin/lttng-clock-plugin-test.c index f0ae2211..d35ecc4a 100644 --- a/tests/clock-plugin/lttng-clock-plugin-test.c +++ b/tests/clock-plugin/lttng-clock-plugin-test.c @@ -10,8 +10,8 @@ #include #include -#include -#include /* From lttng-modules */ +#include +#include /* From lttng-modules */ static u64 trace_clock_read64_example(void) { diff --git a/tests/probes/lttng-test.c b/tests/probes/lttng-test.c index 7654d1fb..5f29fb92 100644 --- a/tests/probes/lttng-test.c +++ b/tests/probes/lttng-test.c @@ -13,8 +13,8 @@ #include #include -#include -#include +#include +#include #define TP_MODULE_NOAUTOLOAD #define LTTNG_PACKAGE_BUILD diff --git a/wrapper/trace-clock.c b/trace-clock.c similarity index 100% rename from wrapper/trace-clock.c rename to trace-clock.c -- 2.34.1