[lttng-ust.git] / libringbuffer / frontend_types.h

#ifndef _LINUX_RING_BUFFER_FRONTEND_TYPES_H
#define _LINUX_RING_BUFFER_FRONTEND_TYPES_H

/*
 * linux/ringbuffer/frontend_types.h
 *
 * (C) Copyright 2005-2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * Ring Buffer Library Synchronization Header (types).
 *
 * Author:
 *	Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * See ring_buffer_frontend.c for more information on wait-free algorithms.
 *
 * Dual LGPL v2.1/GPL v2 license.
 */

#include "config.h"
#include "backend_types.h"
#include "../lib_prio_heap/lttng_prio_heap.h"	/* For per-CPU read-side iterator */

/*
 * A switch is done during tracing or as a final flush after tracing (so it
 * won't write in the new sub-buffer).
 */
enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };

/* channel-level read-side iterator */
struct channel_iter {
	/* Prio heap of buffers. Lowest timestamps at the top. */
	struct lttng_ptr_heap heap;	/* Heap of struct lib_ring_buffer ptrs */
	struct list_head empty_head;	/* Empty buffers linked-list head */
	int read_open;			/* Opened for reading ? */
	u64 last_qs;			/* Last quiescent state timestamp */
	u64 last_timestamp;		/* Last timestamp (for WARN_ON) */
	int last_cpu;			/* Last timestamp cpu */
	/*
	 * read() file operation state.
	 */
	unsigned long len_left;
};

/* channel: collection of per-cpu ring buffers. */
struct channel {
	atomic_t record_disabled;
	unsigned long commit_count_mask;	/*
						 * Commit count mask, removing
						 * the MSBs corresponding to
						 * bits used to represent the
						 * subbuffer index.
						 */

	struct channel_backend backend;		/* Associated backend */

	unsigned long switch_timer_interval;	/* Buffer flush (jiffies) */
	unsigned long read_timer_interval;	/* Reader wakeup (jiffies) */
	struct notifier_block cpu_hp_notifier;	/* CPU hotplug notifier */
	struct notifier_block tick_nohz_notifier; /* CPU nohz notifier */
	struct notifier_block hp_iter_notifier;	/* hotplug iterator notifier */
	int cpu_hp_enable:1;			/* Enable CPU hotplug notif. */
	int hp_iter_enable:1;			/* Enable hp iter notif. */
	wait_queue_head_t read_wait;		/* reader wait queue */
	wait_queue_head_t hp_wait;		/* CPU hotplug wait queue */
	int finalized;				/* Has channel been finalized */
	struct channel_iter iter;		/* Channel read-side iterator */
	struct kref ref;			/* Reference count */
};

/* Per-subbuffer commit counters used on the hot path */
struct commit_counters_hot {
	union v_atomic cc;		/* Commit counter */
	union v_atomic seq;		/* Consecutive commits */
};

/* Per-subbuffer commit counters used only on cold paths */
struct commit_counters_cold {
	union v_atomic cc_sb;		/* Incremented _once_ at sb switch */
};

/* Per-buffer read iterator */
struct lib_ring_buffer_iter {
	u64 timestamp;			/* Current record timestamp */
	size_t header_len;		/* Current record header length */
	size_t payload_len;		/* Current record payload length */

	struct list_head empty_node;	/* Linked list of empty buffers */
	unsigned long consumed, read_offset, data_size;
	enum {
		ITER_GET_SUBBUF = 0,
		ITER_TEST_RECORD,
		ITER_NEXT_RECORD,
		ITER_PUT_SUBBUF,
	} state;
	int allocated:1;
	int read_open:1;		/* Opened for reading ? */
};

/* ring buffer state */
struct lib_ring_buffer {
	/* First 32 bytes cache-hot cacheline */
	union v_atomic offset;		/* Current offset in the buffer */
	struct commit_counters_hot *commit_hot;
					/* Commit count per sub-buffer */
	atomic_long_t consumed;		/*
					 * Current offset in the buffer
					 * standard atomic access (shared)
					 */
	atomic_t record_disabled;
	/* End of first 32 bytes cacheline */
	union v_atomic last_tsc;	/*
					 * Last timestamp written in the buffer.
					 */

	struct lib_ring_buffer_backend backend;	/* Associated backend */

	struct commit_counters_cold *commit_cold;
					/* Commit count per sub-buffer */
	atomic_long_t active_readers;	/*
					 * Active readers count
					 * standard atomic access (shared)
					 */
					/* Dropped records */
	union v_atomic records_lost_full;	/* Buffer full */
	union v_atomic records_lost_wrap;	/* Nested wrap-around */
	union v_atomic records_lost_big;	/* Events too big */
	union v_atomic records_count;	/* Number of records written */
	union v_atomic records_overrun;	/* Number of overwritten records */
	wait_queue_head_t read_wait;	/* reader buffer-level wait queue */
	int finalized;			/* buffer has been finalized */
	struct timer_list switch_timer;	/* timer for periodical switch */
	struct timer_list read_timer;	/* timer for read poll */
	raw_spinlock_t raw_tick_nohz_spinlock;	/* nohz entry lock/trylock */
	struct lib_ring_buffer_iter iter;	/* read-side iterator */
	unsigned long get_subbuf_consumed;	/* Read-side consumed */
	unsigned long prod_snapshot;	/* Producer count snapshot */
	unsigned long cons_snapshot;	/* Consumer count snapshot */
	int get_subbuf:1;		/* Sub-buffer being held by reader */
	int switch_timer_enabled:1;	/* Protected by ring_buffer_nohz_lock */
	int read_timer_enabled:1;	/* Protected by ring_buffer_nohz_lock */
};

static inline
void *channel_get_private(struct channel *chan)
{
	return chan->backend.priv;
}

/*
 * Issue warnings and disable channels upon internal error.
 * Can receive struct lib_ring_buffer or struct lib_ring_buffer_backend
 * parameters.
 */
#define CHAN_WARN_ON(c, cond)						\
	({								\
		struct channel *__chan;					\
		int _____ret = unlikely(cond);				\
		if (_____ret) {						\
			if (__same_type(*(c), struct channel_backend))	\
				__chan = container_of((void *) (c),	\
							struct channel, \
							backend);	\
			else if (__same_type(*(c), struct channel))	\
				__chan = (void *) (c);			\
			else						\
				BUG_ON(1);				\
			atomic_inc(&__chan->record_disabled);		\
			WARN_ON(1);					\
		}							\
		_____ret;						\
	})

#endif /* _LINUX_RING_BUFFER_FRONTEND_TYPES_H */
Commit	Line	Data
852c2936 MD	1	#ifndef _LINUX_RING_BUFFER_FRONTEND_TYPES_H
	2	#define _LINUX_RING_BUFFER_FRONTEND_TYPES_H
	3
	4	/*
	5	* linux/ringbuffer/frontend_types.h
	6	*
	7	* (C) Copyright 2005-2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	8	*
	9	* Ring Buffer Library Synchronization Header (types).
	10	*
	11	* Author:
	12	* Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	13	*
	14	* See ring_buffer_frontend.c for more information on wait-free algorithms.
	15	*
	16	* Dual LGPL v2.1/GPL v2 license.
	17	*/
	18
4931a13e MD	19	#include "config.h"
	20	#include "backend_types.h"
	21	#include "../lib_prio_heap/lttng_prio_heap.h" /* For per-CPU read-side iterator */
852c2936 MD	22
	23	/*
	24	* A switch is done during tracing or as a final flush after tracing (so it
	25	* won't write in the new sub-buffer).
	26	*/
	27	enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };
	28
	29	/* channel-level read-side iterator */
	30	struct channel_iter {
	31	/* Prio heap of buffers. Lowest timestamps at the top. */
	32	struct lttng_ptr_heap heap; /* Heap of struct lib_ring_buffer ptrs */
	33	struct list_head empty_head; /* Empty buffers linked-list head */
	34	int read_open; /* Opened for reading ? */
	35	u64 last_qs; /* Last quiescent state timestamp */
	36	u64 last_timestamp; /* Last timestamp (for WARN_ON) */
	37	int last_cpu; /* Last timestamp cpu */
	38	/*
	39	* read() file operation state.
	40	*/
	41	unsigned long len_left;
	42	};
	43
	44	/* channel: collection of per-cpu ring buffers. */
	45	struct channel {
	46	atomic_t record_disabled;
	47	unsigned long commit_count_mask; /*
	48	* Commit count mask, removing
	49	* the MSBs corresponding to
	50	* bits used to represent the
	51	* subbuffer index.
	52	*/
	53
	54	struct channel_backend backend; /* Associated backend */
	55
	56	unsigned long switch_timer_interval; /* Buffer flush (jiffies) */
	57	unsigned long read_timer_interval; /* Reader wakeup (jiffies) */
	58	struct notifier_block cpu_hp_notifier; /* CPU hotplug notifier */
	59	struct notifier_block tick_nohz_notifier; /* CPU nohz notifier */
	60	struct notifier_block hp_iter_notifier; /* hotplug iterator notifier */
	61	int cpu_hp_enable:1; /* Enable CPU hotplug notif. */
	62	int hp_iter_enable:1; /* Enable hp iter notif. */
	63	wait_queue_head_t read_wait; /* reader wait queue */
	64	wait_queue_head_t hp_wait; /* CPU hotplug wait queue */
	65	int finalized; /* Has channel been finalized */
	66	struct channel_iter iter; /* Channel read-side iterator */
	67	struct kref ref; /* Reference count */
	68	};
	69
	70	/* Per-subbuffer commit counters used on the hot path */
	71	struct commit_counters_hot {
	72	union v_atomic cc; /* Commit counter */
	73	union v_atomic seq; /* Consecutive commits */
	74	};
	75
	76	/* Per-subbuffer commit counters used only on cold paths */
	77	struct commit_counters_cold {
	78	union v_atomic cc_sb; /* Incremented _once_ at sb switch */
	79	};
	80
	81	/* Per-buffer read iterator */
	82	struct lib_ring_buffer_iter {
	83	u64 timestamp; /* Current record timestamp */
	84	size_t header_len; /* Current record header length */
	85	size_t payload_len; /* Current record payload length */
86
87	struct list_head empty_node; /* Linked list of empty buffers */
88	unsigned long consumed, read_offset, data_size;
89	enum {
90	ITER_GET_SUBBUF = 0,
91	ITER_TEST_RECORD,
92	ITER_NEXT_RECORD,
93	ITER_PUT_SUBBUF,
94	} state;
95	int allocated:1;
96	int read_open:1; /* Opened for reading ? */
97	};
98
99	/* ring buffer state */
100	struct lib_ring_buffer {
101	/* First 32 bytes cache-hot cacheline */
102	union v_atomic offset; /* Current offset in the buffer */
103	struct commit_counters_hot *commit_hot;
104	/* Commit count per sub-buffer */
105	atomic_long_t consumed; /*
106	* Current offset in the buffer
107	* standard atomic access (shared)
108	*/
109	atomic_t record_disabled;
110	/* End of first 32 bytes cacheline */
111	union v_atomic last_tsc; /*
112	* Last timestamp written in the buffer.
113	*/
114
115	struct lib_ring_buffer_backend backend; /* Associated backend */
116
117	struct commit_counters_cold *commit_cold;
118	/* Commit count per sub-buffer */
119	atomic_long_t active_readers; /*
120	* Active readers count
121	* standard atomic access (shared)
122	*/
123	/* Dropped records */
124	union v_atomic records_lost_full; /* Buffer full */
125	union v_atomic records_lost_wrap; /* Nested wrap-around */
126	union v_atomic records_lost_big; /* Events too big */
127	union v_atomic records_count; /* Number of records written */
128	union v_atomic records_overrun; /* Number of overwritten records */
129	wait_queue_head_t read_wait; /* reader buffer-level wait queue */
130	int finalized; /* buffer has been finalized */
131	struct timer_list switch_timer; /* timer for periodical switch */
132	struct timer_list read_timer; /* timer for read poll */
133	raw_spinlock_t raw_tick_nohz_spinlock; /* nohz entry lock/trylock */
134	struct lib_ring_buffer_iter iter; /* read-side iterator */
135	unsigned long get_subbuf_consumed; /* Read-side consumed */
136	unsigned long prod_snapshot; /* Producer count snapshot */
137	unsigned long cons_snapshot; /* Consumer count snapshot */
138	int get_subbuf:1; /* Sub-buffer being held by reader */
139	int switch_timer_enabled:1; /* Protected by ring_buffer_nohz_lock */
140	int read_timer_enabled:1; /* Protected by ring_buffer_nohz_lock */
141	};
142
143	static inline
144	void channel_get_private(struct channel chan)
145	{
146	return chan->backend.priv;
147	}
148
149	/*
150	* Issue warnings and disable channels upon internal error.
151	* Can receive struct lib_ring_buffer or struct lib_ring_buffer_backend
152	* parameters.
153	*/
154	#define CHAN_WARN_ON(c, cond) \
155	({ \
156	struct channel *__chan; \
157	int _____ret = unlikely(cond); \
158	if (_____ret) { \
159	if (__same_type(*(c), struct channel_backend)) \
160	__chan = container_of((void *) (c), \
161	struct channel, \
162	backend); \
163	else if (__same_type(*(c), struct channel)) \
164	__chan = (void *) (c); \
165	else \
166	BUG_ON(1); \
167	atomic_inc(&__chan->record_disabled); \
168	WARN_ON(1); \
169	} \
170	_____ret; \
171	})
172
173	#endif /* _LINUX_RING_BUFFER_FRONTEND_TYPES_H */