[lttng-ust.git] / libringbuffer / frontend_types.h

#ifndef _LTTNG_RING_BUFFER_FRONTEND_TYPES_H
#define _LTTNG_RING_BUFFER_FRONTEND_TYPES_H

/*
 * libringbuffer/frontend_types.h
 *
 * Ring Buffer Library Synchronization Header (types).
 *
 * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; only
 * version 2.1 of the License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * Author:
 *	Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * See ring_buffer_frontend.c for more information on wait-free algorithms.
 */

#include <stdint.h>
#include <string.h>
#include <time.h>	/* for timer_t */

#include <urcu/list.h>
#include <urcu/uatomic.h>

#include <lttng/ringbuffer-config.h>
#include <usterr-signal-safe.h>
#include "backend_types.h"
#include "shm_internal.h"
#include "shm_types.h"
#include "vatomic.h"

/*
 * A switch is done during tracing or as a final flush after tracing (so it
 * won't write in the new sub-buffer).
 */
enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };

/* channel: collection of per-cpu ring buffers. */
#define RB_CHANNEL_PADDING		32
struct channel {
	int record_disabled;
	unsigned long commit_count_mask;	/*
						 * Commit count mask, removing
						 * the MSBs corresponding to
						 * bits used to represent the
						 * subbuffer index.
						 */

	unsigned long switch_timer_interval;	/* Buffer flush (us) */
	timer_t switch_timer;
	int switch_timer_enabled;

	unsigned long read_timer_interval;	/* Reader wakeup (us) */
	timer_t read_timer;
	int read_timer_enabled;

	int finalized;				/* Has channel been finalized */
	size_t priv_data_offset;
	unsigned int nr_streams;		/* Number of streams */
	struct lttng_ust_shm_handle *handle;
	/* Extended options. */
	union {
		struct {
			int32_t blocking_timeout_ms;
		} s;
		char padding[RB_CHANNEL_PADDING];
	} u;
	/*
	 * Associated backend contains a variable-length array. Needs to
	 * be last member.
	 */
	struct channel_backend backend;		/* Associated backend */
} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));

/* Per-subbuffer commit counters used on the hot path */
#define RB_COMMIT_COUNT_HOT_PADDING	16
struct commit_counters_hot {
	union v_atomic cc;		/* Commit counter */
	union v_atomic seq;		/* Consecutive commits */
	char padding[RB_COMMIT_COUNT_HOT_PADDING];
} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));

/* Per-subbuffer commit counters used only on cold paths */
#define RB_COMMIT_COUNT_COLD_PADDING	24
struct commit_counters_cold {
	union v_atomic cc_sb;		/* Incremented _once_ at sb switch */
	char padding[RB_COMMIT_COUNT_COLD_PADDING];
} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));

/* ring buffer state */
#define RB_CRASH_DUMP_ABI_LEN		256
#define RB_RING_BUFFER_PADDING		60

#define RB_CRASH_DUMP_ABI_MAGIC_LEN	16

/*
 * The 128-bit magic number is xor'd in the process data so it does not
 * cause a false positive when searching for buffers by scanning memory.
 * The actual magic number is:
 *   0x17, 0x7B, 0xF1, 0x77, 0xBF, 0x17, 0x7B, 0xF1,
 *   0x77, 0xBF, 0x17, 0x7B, 0xF1, 0x77, 0xBF, 0x17,
 */
#define RB_CRASH_DUMP_ABI_MAGIC_XOR					\
	{								\
		0x17 ^ 0xFF, 0x7B ^ 0xFF, 0xF1 ^ 0xFF, 0x77 ^ 0xFF,	\
		0xBF ^ 0xFF, 0x17 ^ 0xFF, 0x7B ^ 0xFF, 0xF1 ^ 0xFF,	\
		0x77 ^ 0xFF, 0xBF ^ 0xFF, 0x17 ^ 0xFF, 0x7B ^ 0xFF,	\
		0xF1 ^ 0xFF, 0x77 ^ 0xFF, 0xBF ^ 0xFF, 0x17 ^ 0xFF,	\
	}

#define RB_CRASH_ENDIAN			0x1234

#define RB_CRASH_DUMP_ABI_MAJOR		0
#define RB_CRASH_DUMP_ABI_MINOR		0

enum lttng_crash_type {
	LTTNG_CRASH_TYPE_UST = 0,
	LTTNG_CRASH_TYPE_KERNEL = 1,
};

struct lttng_crash_abi {
	uint8_t magic[RB_CRASH_DUMP_ABI_MAGIC_LEN];
	uint64_t mmap_length;	/* Overall lenght of crash record */
	uint16_t endian;	/*
				 * { 0x12, 0x34 }: big endian
				 * { 0x34, 0x12 }: little endian
				 */
	uint16_t major;		/* Major number. */
	uint16_t minor;		/* Minor number. */
	uint8_t word_size;	/* Word size (bytes). */
	uint8_t layout_type;	/* enum lttng_crash_type */

	struct {
		uint32_t prod_offset;
		uint32_t consumed_offset;
		uint32_t commit_hot_array;
		uint32_t commit_hot_seq;
		uint32_t buf_wsb_array;
		uint32_t buf_wsb_id;
		uint32_t sb_array;
		uint32_t sb_array_shmp_offset;
		uint32_t sb_backend_p_offset;
		uint32_t content_size;
		uint32_t packet_size;
	} __attribute__((packed)) offset;
	struct {
		uint8_t prod_offset;
		uint8_t consumed_offset;
		uint8_t commit_hot_seq;
		uint8_t buf_wsb_id;
		uint8_t sb_array_shmp_offset;
		uint8_t sb_backend_p_offset;
		uint8_t content_size;
		uint8_t packet_size;
	} __attribute__((packed)) length;
	struct {
		uint32_t commit_hot_array;
		uint32_t buf_wsb_array;
		uint32_t sb_array;
	} __attribute__((packed)) stride;

	uint64_t buf_size;	/* Size of the buffer */
	uint64_t subbuf_size;	/* Sub-buffer size */
	uint64_t num_subbuf;	/* Number of sub-buffers for writer */
	uint32_t mode;		/* Buffer mode: 0: overwrite, 1: discard */
} __attribute__((packed));

struct lttng_ust_lib_ring_buffer {
	/* First 32 bytes are for the buffer crash dump ABI */
	struct lttng_crash_abi crash_abi;

	/* 32 bytes cache-hot cacheline */
	union v_atomic __attribute__((aligned(32))) offset;
					/* Current offset in the buffer */
	DECLARE_SHMP(struct commit_counters_hot, commit_hot);
					/* Commit count per sub-buffer */
	long consumed;			/*
					 * Current offset in the buffer
					 * standard atomic access (shared)
					 */
	int record_disabled;
	/* End of cache-hot 32 bytes cacheline */

	union v_atomic last_tsc;	/*
					 * Last timestamp written in the buffer.
					 */

	struct lttng_ust_lib_ring_buffer_backend backend;
					/* Associated backend */

	DECLARE_SHMP(struct commit_counters_cold, commit_cold);
					/* Commit count per sub-buffer */
	DECLARE_SHMP(uint64_t, ts_end);	/*
					 * timestamp_end per sub-buffer.
					 * Time is sampled by the
					 * switch_*_end() callbacks
					 * which are the last space
					 * reservation performed in the
					 * sub-buffer before it can be
					 * fully committed and
					 * delivered. This time value is
					 * then read by the deliver
					 * callback, performed by the
					 * last commit before the buffer
					 * becomes readable.
					 */
	long active_readers;		/*
					 * Active readers count
					 * standard atomic access (shared)
					 */
					/* Dropped records */
	union v_atomic records_lost_full;	/* Buffer full */
	union v_atomic records_lost_wrap;	/* Nested wrap-around */
	union v_atomic records_lost_big;	/* Events too big */
	union v_atomic records_count;	/* Number of records written */
	union v_atomic records_overrun;	/* Number of overwritten records */
	//wait_queue_head_t read_wait;	/* reader buffer-level wait queue */
	int finalized;			/* buffer has been finalized */
	unsigned long get_subbuf_consumed;	/* Read-side consumed */
	unsigned long prod_snapshot;	/* Producer count snapshot */
	unsigned long cons_snapshot;	/* Consumer count snapshot */
	unsigned int get_subbuf:1;	/* Sub-buffer being held by reader */
	/* shmp pointer to self */
	DECLARE_SHMP(struct lttng_ust_lib_ring_buffer, self);
	char padding[RB_RING_BUFFER_PADDING];
} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));

static inline
void *channel_get_private(struct channel *chan)
{
	return ((char *) chan) + chan->priv_data_offset;
}

#ifndef __rb_same_type
#define __rb_same_type(a, b)	__builtin_types_compatible_p(typeof(a), typeof(b))
#endif

/*
 * Issue warnings and disable channels upon internal error.
 * Can receive struct lttng_ust_lib_ring_buffer or struct lttng_ust_lib_ring_buffer_backend
 * parameters.
 */
#define CHAN_WARN_ON(c, cond)						\
	({								\
		struct channel *__chan;					\
		int _____ret = caa_unlikely(cond);				\
		if (_____ret) {						\
			if (__rb_same_type(*(c), struct channel_backend))	\
				__chan = caa_container_of((void *) (c),	\
							struct channel, \
							backend);	\
			else if (__rb_same_type(*(c), struct channel))	\
				__chan = (void *) (c);			\
			else						\
				BUG_ON(1);				\
			uatomic_inc(&__chan->record_disabled);		\
			WARN_ON(1);					\
		}							\
		_____ret = _____ret; /* For clang "unused result". */	\
	})

#endif /* _LTTNG_RING_BUFFER_FRONTEND_TYPES_H */
Commit	Line	Data
e92f3e28 MD	1	#ifndef _LTTNG_RING_BUFFER_FRONTEND_TYPES_H
e92f3e28 MD	2	#define _LTTNG_RING_BUFFER_FRONTEND_TYPES_H
852c2936 MD	3
852c2936 MD	4	/*
e92f3e28	5	* libringbuffer/frontend_types.h
852c2936 MD	6	*
	7	* Ring Buffer Library Synchronization Header (types).
	8	*
e92f3e28 MD	9	* Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; only
	14	* version 2.1 of the License.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
	22	* License along with this library; if not, write to the Free Software
	23	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	24	*
852c2936 MD	25	* Author:
	26	* Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	27	*
	28	* See ring_buffer_frontend.c for more information on wait-free algorithms.
852c2936 MD	29	*/
852c2936 MD	30
fb31eb73	31	#include <stdint.h>
a6352fd4	32	#include <string.h>
03d2d293	33	#include <time.h> /* for timer_t */
a6352fd4	34
14641deb MD	35	#include <urcu/list.h>
14641deb MD	36	#include <urcu/uatomic.h>
14641deb	37
4318ae1b	38	#include <lttng/ringbuffer-config.h>
44c72f10	39	#include <usterr-signal-safe.h>
4931a13e	40	#include "backend_types.h"
1d498196	41	#include "shm_internal.h"
03d2d293	42	#include "shm_types.h"
a3bb4b27	43	#include "vatomic.h"
852c2936 MD	44
	45	/*
	46	* A switch is done during tracing or as a final flush after tracing (so it
	47	* won't write in the new sub-buffer).
	48	*/
	49	enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };
	50
852c2936	51	/* channel: collection of per-cpu ring buffers. */
74d81a6c	52	#define RB_CHANNEL_PADDING 32
852c2936	53	struct channel {
14641deb	54	int record_disabled;
852c2936 MD	55	unsigned long commit_count_mask; /*
	56	* Commit count mask, removing
	57	* the MSBs corresponding to
	58	* bits used to represent the
	59	* subbuffer index.
	60	*/
	61
03d2d293 MD	62	unsigned long switch_timer_interval; /* Buffer flush (us) */
	63	timer_t switch_timer;
	64	int switch_timer_enabled;
	65
	66	unsigned long read_timer_interval; /* Reader wakeup (us) */
34a91bdb MD	67	timer_t read_timer;
	68	int read_timer_enabled;
	69
852c2936	70	int finalized; /* Has channel been finalized */
a3f61e7f	71	size_t priv_data_offset;
74d81a6c	72	unsigned int nr_streams; /* Number of streams */
03d2d293	73	struct lttng_ust_shm_handle *handle;
b2c5f61a MD	74	/* Extended options. */
	75	union {
	76	struct {
	77	int32_t blocking_timeout_ms;
	78	} s;
	79	char padding[RB_CHANNEL_PADDING];
	80	} u;
de85e7c3 MD	81	/*
	82	* Associated backend contains a variable-length array. Needs to
	83	* be last member.
	84	*/
	85	struct channel_backend backend; /* Associated backend */
b728d87e	86	} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
852c2936 MD	87
852c2936 MD	88	/* Per-subbuffer commit counters used on the hot path */
3c8964ba	89	#define RB_COMMIT_COUNT_HOT_PADDING 16
852c2936 MD	90	struct commit_counters_hot {
	91	union v_atomic cc; /* Commit counter */
	92	union v_atomic seq; /* Consecutive commits */
3c8964ba	93	char padding[RB_COMMIT_COUNT_HOT_PADDING];
b728d87e	94	} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
852c2936 MD	95
852c2936 MD	96	/* Per-subbuffer commit counters used only on cold paths */
3c8964ba	97	#define RB_COMMIT_COUNT_COLD_PADDING 24
852c2936 MD	98	struct commit_counters_cold {
852c2936 MD	99	union v_atomic cc_sb; /* Incremented _once_ at sb switch */
3c8964ba	100	char padding[RB_COMMIT_COUNT_COLD_PADDING];
b728d87e	101	} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
852c2936	102
852c2936	103	/* ring buffer state */
a9ff648c MD	104	#define RB_CRASH_DUMP_ABI_LEN 256
	105	#define RB_RING_BUFFER_PADDING 60
	106
	107	#define RB_CRASH_DUMP_ABI_MAGIC_LEN 16
	108
	109	/*
	110	* The 128-bit magic number is xor'd in the process data so it does not
	111	* cause a false positive when searching for buffers by scanning memory.
	112	* The actual magic number is:
	113	* 0x17, 0x7B, 0xF1, 0x77, 0xBF, 0x17, 0x7B, 0xF1,
	114	* 0x77, 0xBF, 0x17, 0x7B, 0xF1, 0x77, 0xBF, 0x17,
	115	*/
	116	#define RB_CRASH_DUMP_ABI_MAGIC_XOR \
	117	{ \
	118	0x17 ^ 0xFF, 0x7B ^ 0xFF, 0xF1 ^ 0xFF, 0x77 ^ 0xFF, \
	119	0xBF ^ 0xFF, 0x17 ^ 0xFF, 0x7B ^ 0xFF, 0xF1 ^ 0xFF, \
	120	0x77 ^ 0xFF, 0xBF ^ 0xFF, 0x17 ^ 0xFF, 0x7B ^ 0xFF, \
	121	0xF1 ^ 0xFF, 0x77 ^ 0xFF, 0xBF ^ 0xFF, 0x17 ^ 0xFF, \
	122	}
	123
	124	#define RB_CRASH_ENDIAN 0x1234
	125
	126	#define RB_CRASH_DUMP_ABI_MAJOR 0
	127	#define RB_CRASH_DUMP_ABI_MINOR 0
	128
	129	enum lttng_crash_type {
	130	LTTNG_CRASH_TYPE_UST = 0,
	131	LTTNG_CRASH_TYPE_KERNEL = 1,
	132	};
	133
	134	struct lttng_crash_abi {
	135	uint8_t magic[RB_CRASH_DUMP_ABI_MAGIC_LEN];
	136	uint64_t mmap_length; /* Overall lenght of crash record */
	137	uint16_t endian; /*
	138	* { 0x12, 0x34 }: big endian
	139	* { 0x34, 0x12 }: little endian
	140	*/
	141	uint16_t major; /* Major number. */
	142	uint16_t minor; /* Minor number. */
	143	uint8_t word_size; /* Word size (bytes). */
	144	uint8_t layout_type; /* enum lttng_crash_type */
	145
	146	struct {
	147	uint32_t prod_offset;
	148	uint32_t consumed_offset;
	149	uint32_t commit_hot_array;
	150	uint32_t commit_hot_seq;
	151	uint32_t buf_wsb_array;
	152	uint32_t buf_wsb_id;
	153	uint32_t sb_array;
	154	uint32_t sb_array_shmp_offset;
	155	uint32_t sb_backend_p_offset;
	156	uint32_t content_size;
	157	uint32_t packet_size;
	158	} __attribute__((packed)) offset;
	159	struct {
	160	uint8_t prod_offset;
	161	uint8_t consumed_offset;
	162	uint8_t commit_hot_seq;
	163	uint8_t buf_wsb_id;
	164	uint8_t sb_array_shmp_offset;
	165	uint8_t sb_backend_p_offset;
	166	uint8_t content_size;
	167	uint8_t packet_size;
168	} __attribute__((packed)) length;
169	struct {
170	uint32_t commit_hot_array;
171	uint32_t buf_wsb_array;
172	uint32_t sb_array;
173	} __attribute__((packed)) stride;
174
175	uint64_t buf_size; /* Size of the buffer */
176	uint64_t subbuf_size; /* Sub-buffer size */
177	uint64_t num_subbuf; /* Number of sub-buffers for writer */
178	uint32_t mode; /* Buffer mode: 0: overwrite, 1: discard */
179	} __attribute__((packed));
180
4cfec15c	181	struct lttng_ust_lib_ring_buffer {
a9ff648c MD	182	/* First 32 bytes are for the buffer crash dump ABI */
	183	struct lttng_crash_abi crash_abi;
	184
	185	/* 32 bytes cache-hot cacheline */
	186	union v_atomic __attribute__((aligned(32))) offset;
	187	/* Current offset in the buffer */
a6352fd4	188	DECLARE_SHMP(struct commit_counters_hot, commit_hot);
852c2936	189	/* Commit count per sub-buffer */
14641deb	190	long consumed; /*
852c2936 MD	191	* Current offset in the buffer
	192	* standard atomic access (shared)
	193	*/
14641deb	194	int record_disabled;
a9ff648c MD	195	/* End of cache-hot 32 bytes cacheline */
a9ff648c MD	196
852c2936 MD	197	union v_atomic last_tsc; /*
	198	* Last timestamp written in the buffer.
	199	*/
	200
a9ff648c MD	201	struct lttng_ust_lib_ring_buffer_backend backend;
a9ff648c MD	202	/* Associated backend */
852c2936	203
a6352fd4	204	DECLARE_SHMP(struct commit_counters_cold, commit_cold);
852c2936	205	/* Commit count per sub-buffer */
6c737d05 MD	206	DECLARE_SHMP(uint64_t, ts_end); /*
	207	* timestamp_end per sub-buffer.
	208	* Time is sampled by the
	209	* switch_*_end() callbacks
	210	* which are the last space
	211	* reservation performed in the
	212	* sub-buffer before it can be
	213	* fully committed and
	214	* delivered. This time value is
	215	* then read by the deliver
	216	* callback, performed by the
	217	* last commit before the buffer
	218	* becomes readable.
	219	*/
14641deb	220	long active_readers; /*
852c2936 MD	221	* Active readers count
	222	* standard atomic access (shared)
	223	*/
	224	/* Dropped records */
	225	union v_atomic records_lost_full; /* Buffer full */
	226	union v_atomic records_lost_wrap; /* Nested wrap-around */
	227	union v_atomic records_lost_big; /* Events too big */
	228	union v_atomic records_count; /* Number of records written */
	229	union v_atomic records_overrun; /* Number of overwritten records */
14641deb	230	//wait_queue_head_t read_wait; /* reader buffer-level wait queue */
852c2936	231	int finalized; /* buffer has been finalized */
852c2936 MD	232	unsigned long get_subbuf_consumed; /* Read-side consumed */
	233	unsigned long prod_snapshot; /* Producer count snapshot */
	234	unsigned long cons_snapshot; /* Consumer count snapshot */
34a91bdb	235	unsigned int get_subbuf:1; /* Sub-buffer being held by reader */
5d61a504	236	/* shmp pointer to self */
4cfec15c	237	DECLARE_SHMP(struct lttng_ust_lib_ring_buffer, self);
3c8964ba	238	char padding[RB_RING_BUFFER_PADDING];
b728d87e	239	} __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
852c2936 MD	240
	241	static inline
	242	void channel_get_private(struct channel chan)
	243	{
a3f61e7f	244	return ((char *) chan) + chan->priv_data_offset;
852c2936 MD	245	}
852c2936 MD	246
0d4aa2df MD	247	#ifndef __rb_same_type
	248	#define __rb_same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))
	249	#endif
	250
852c2936 MD	251	/*
852c2936 MD	252	* Issue warnings and disable channels upon internal error.
4cfec15c	253	* Can receive struct lttng_ust_lib_ring_buffer or struct lttng_ust_lib_ring_buffer_backend
852c2936 MD	254	* parameters.
	255	*/
	256	#define CHAN_WARN_ON(c, cond) \
	257	({ \
	258	struct channel *__chan; \
b5a3dfa5	259	int _____ret = caa_unlikely(cond); \
852c2936	260	if (_____ret) { \
0d4aa2df	261	if (__rb_same_type(*(c), struct channel_backend)) \
14641deb	262	__chan = caa_container_of((void *) (c), \
852c2936 MD	263	struct channel, \
852c2936 MD	264	backend); \
0d4aa2df	265	else if (__rb_same_type(*(c), struct channel)) \
852c2936 MD	266	__chan = (void *) (c); \
	267	else \
	268	BUG_ON(1); \
14641deb	269	uatomic_inc(&__chan->record_disabled); \
852c2936 MD	270	WARN_ON(1); \
852c2936 MD	271	} \
257ecc17	272	_____ret = _____ret; /* For clang "unused result". */ \
852c2936 MD	273	})
852c2936 MD	274
e92f3e28	275	#endif /* _LTTNG_RING_BUFFER_FRONTEND_TYPES_H */