[lttng-modules.git] / lib / ringbuffer / backend.h

#ifndef _LIB_RING_BUFFER_BACKEND_H
#define _LIB_RING_BUFFER_BACKEND_H

/*
 * lib/ringbuffer/backend.h
 *
 * Ring buffer backend (API).
 *
 * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; only
 * version 2.1 of the License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
 * the reader in flight recorder mode.
 */

#include <linux/types.h>
#include <linux/sched.h>
#include <linux/timer.h>
#include <linux/wait.h>
#include <linux/poll.h>
#include <linux/list.h>
#include <linux/fs.h>
#include <linux/mm.h>

/* Internal helpers */
#include "../../wrapper/ringbuffer/backend_internal.h"
#include "../../wrapper/ringbuffer/frontend_internal.h"

/* Ring buffer backend API */

/* Ring buffer backend access (read/write) */

extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb,
				   size_t offset, void *dest, size_t len);

extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb,
					  size_t offset, void __user *dest,
					  size_t len);

extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb,
				     size_t offset, void *dest, size_t len);

extern struct page **
lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend *bufb, size_t offset,
			      void ***virt);

/*
 * Return the address where a given offset is located.
 * Should be used to get the current subbuffer header pointer. Given we know
 * it's never on a page boundary, it's safe to write directly to this address,
 * as long as the write is never bigger than a page size.
 */
extern void *
lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb,
			       size_t offset);
extern void *
lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb,
				    size_t offset);

/**
 * lib_ring_buffer_write - write data to a buffer backend
 * @config : ring buffer instance configuration
 * @ctx: ring buffer context. (input arguments only)
 * @src : source pointer to copy from
 * @len : length of data to copy
 *
 * This function copies "len" bytes of data from a source pointer to a buffer
 * backend, at the current context offset. This is more or less a buffer
 * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
 * if copy is crossing a page boundary.
 */
static inline
void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
			   struct lib_ring_buffer_ctx *ctx,
			   const void *src, size_t len)
{
	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	struct channel_backend *chanb = &ctx->chan->backend;
	size_t sbidx, index;
	size_t offset = ctx->buf_offset;
	ssize_t pagecpy;
	struct lib_ring_buffer_backend_pages *rpages;
	unsigned long sb_bindex, id;

	offset &= chanb->buf_size - 1;
	sbidx = offset >> chanb->subbuf_size_order;
	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	id = bufb->buf_wsb[sbidx].id;
	sb_bindex = subbuffer_id_get_index(config, id);
	rpages = bufb->array[sb_bindex];
	CHAN_WARN_ON(ctx->chan,
		     config->mode == RING_BUFFER_OVERWRITE
		     && subbuffer_id_is_noref(config, id));
	if (likely(pagecpy == len))
		lib_ring_buffer_do_copy(config,
					rpages->p[index].virt
					    + (offset & ~PAGE_MASK),
					src, len);
	else
		_lib_ring_buffer_write(bufb, offset, src, len, 0);
	ctx->buf_offset += len;
}

/**
 * lib_ring_buffer_memset - write len bytes of c to a buffer backend
 * @config : ring buffer instance configuration
 * @bufb : ring buffer backend
 * @offset : offset within the buffer
 * @c : the byte to copy
 * @len : number of bytes to copy
 *
 * This function writes "len" bytes of "c" to a buffer backend, at a specific
 * offset. This is more or less a buffer backend-specific memset() operation.
 * Calls the slow path (_ring_buffer_memset) if write is crossing a page
 * boundary.
 */
static inline
void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config,
			    struct lib_ring_buffer_ctx *ctx, int c, size_t len)
{

	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	struct channel_backend *chanb = &ctx->chan->backend;
	size_t sbidx, index;
	size_t offset = ctx->buf_offset;
	ssize_t pagecpy;
	struct lib_ring_buffer_backend_pages *rpages;
	unsigned long sb_bindex, id;

	offset &= chanb->buf_size - 1;
	sbidx = offset >> chanb->subbuf_size_order;
	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	id = bufb->buf_wsb[sbidx].id;
	sb_bindex = subbuffer_id_get_index(config, id);
	rpages = bufb->array[sb_bindex];
	CHAN_WARN_ON(ctx->chan,
		     config->mode == RING_BUFFER_OVERWRITE
		     && subbuffer_id_is_noref(config, id));
	if (likely(pagecpy == len))
		lib_ring_buffer_do_memset(rpages->p[index].virt
					  + (offset & ~PAGE_MASK),
					  c, len);
	else
		_lib_ring_buffer_memset(bufb, offset, c, len, 0);
	ctx->buf_offset += len;
}

/**
 * lib_ring_buffer_copy_from_user - write userspace data to a buffer backend
 * @config : ring buffer instance configuration
 * @ctx: ring buffer context. (input arguments only)
 * @src : userspace source pointer to copy from
 * @len : length of data to copy
 *
 * This function copies "len" bytes of data from a userspace pointer to a
 * buffer backend, at the current context offset. This is more or less a buffer
 * backend-specific memcpy() operation. Calls the slow path
 * (_ring_buffer_write_from_user) if copy is crossing a page boundary.
 */
static inline
void lib_ring_buffer_copy_from_user(const struct lib_ring_buffer_config *config,
				    struct lib_ring_buffer_ctx *ctx,
				    const void __user *src, size_t len)
{
	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	struct channel_backend *chanb = &ctx->chan->backend;
	size_t sbidx, index;
	size_t offset = ctx->buf_offset;
	ssize_t pagecpy;
	struct lib_ring_buffer_backend_pages *rpages;
	unsigned long sb_bindex, id;
	unsigned long ret;

	offset &= chanb->buf_size - 1;
	sbidx = offset >> chanb->subbuf_size_order;
	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	id = bufb->buf_wsb[sbidx].id;
	sb_bindex = subbuffer_id_get_index(config, id);
	rpages = bufb->array[sb_bindex];
	CHAN_WARN_ON(ctx->chan,
		     config->mode == RING_BUFFER_OVERWRITE
		     && subbuffer_id_is_noref(config, id));

	if (unlikely(!access_ok(VERIFY_READ, src, len)))
		goto fill_buffer;

	if (likely(pagecpy == len)) {
		ret = lib_ring_buffer_do_copy_from_user(
			rpages->p[index].virt + (offset & ~PAGE_MASK),
			src, len);
		if (unlikely(ret > 0)) {
			len -= (pagecpy - ret);
			offset += (pagecpy - ret);
			goto fill_buffer;
		}
	} else {
		_lib_ring_buffer_copy_from_user(bufb, offset, src, len, 0);
	}
	ctx->buf_offset += len;

	return;

fill_buffer:
	/*
	 * In the error path we call the slow path version to avoid
	 * the pollution of static inline code.
	 */
	_lib_ring_buffer_memset(bufb, offset, 0, len, 0);
}

/*
 * This accessor counts the number of unread records in a buffer.
 * It only provides a consistent value if no reads not writes are performed
 * concurrently.
 */
static inline
unsigned long lib_ring_buffer_get_records_unread(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	struct lib_ring_buffer_backend *bufb = &buf->backend;
	struct lib_ring_buffer_backend_pages *pages;
	unsigned long records_unread = 0, sb_bindex, id;
	unsigned int i;

	for (i = 0; i < bufb->chan->backend.num_subbuf; i++) {
		id = bufb->buf_wsb[i].id;
		sb_bindex = subbuffer_id_get_index(config, id);
		pages = bufb->array[sb_bindex];
		records_unread += v_read(config, &pages->records_unread);
	}
	if (config->mode == RING_BUFFER_OVERWRITE) {
		id = bufb->buf_rsb.id;
		sb_bindex = subbuffer_id_get_index(config, id);
		pages = bufb->array[sb_bindex];
		records_unread += v_read(config, &pages->records_unread);
	}
	return records_unread;
}

ssize_t lib_ring_buffer_file_splice_read(struct file *in, loff_t *ppos,
					 struct pipe_inode_info *pipe,
					 size_t len, unsigned int flags);
loff_t lib_ring_buffer_no_llseek(struct file *file, loff_t offset, int origin);

#endif /* _LIB_RING_BUFFER_BACKEND_H */
Commit	Line	Data
886d51a3 MD	1	#ifndef _LIB_RING_BUFFER_BACKEND_H
886d51a3 MD	2	#define _LIB_RING_BUFFER_BACKEND_H
f3bc08c5 MD	3
f3bc08c5 MD	4	/*
886d51a3	5	* lib/ringbuffer/backend.h
f3bc08c5 MD	6	*
	7	* Ring buffer backend (API).
	8	*
886d51a3 MD	9	* Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; only
	14	* version 2.1 of the License.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
	22	* License along with this library; if not, write to the Free Software
	23	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
f3bc08c5 MD	24	*
	25	* Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
	26	* the reader in flight recorder mode.
	27	*/
	28
	29	#include <linux/types.h>
	30	#include <linux/sched.h>
	31	#include <linux/timer.h>
	32	#include <linux/wait.h>
	33	#include <linux/poll.h>
	34	#include <linux/list.h>
	35	#include <linux/fs.h>
	36	#include <linux/mm.h>
	37
	38	/* Internal helpers */
	39	#include "../../wrapper/ringbuffer/backend_internal.h"
	40	#include "../../wrapper/ringbuffer/frontend_internal.h"
	41
	42	/* Ring buffer backend API */
	43
	44	/* Ring buffer backend access (read/write) */
	45
	46	extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb,
	47	size_t offset, void *dest, size_t len);
	48
	49	extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb,
	50	size_t offset, void __user *dest,
	51	size_t len);
	52
	53	extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb,
	54	size_t offset, void *dest, size_t len);
	55
	56	extern struct page **
	57	lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend *bufb, size_t offset,
	58	void ***virt);
	59
	60	/*
	61	* Return the address where a given offset is located.
	62	* Should be used to get the current subbuffer header pointer. Given we know
	63	* it's never on a page boundary, it's safe to write directly to this address,
	64	* as long as the write is never bigger than a page size.
	65	*/
	66	extern void *
	67	lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb,
	68	size_t offset);
	69	extern void *
	70	lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb,
	71	size_t offset);
	72
	73	/**
	74	* lib_ring_buffer_write - write data to a buffer backend
	75	* @config : ring buffer instance configuration
	76	* @ctx: ring buffer context. (input arguments only)
	77	* @src : source pointer to copy from
	78	* @len : length of data to copy
	79	*
	80	* This function copies "len" bytes of data from a source pointer to a buffer
	81	* backend, at the current context offset. This is more or less a buffer
	82	* backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
	83	* if copy is crossing a page boundary.
	84	*/
	85	static inline
	86	void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
	87	struct lib_ring_buffer_ctx *ctx,
88	const void *src, size_t len)
89	{
90	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
91	struct channel_backend *chanb = &ctx->chan->backend;
92	size_t sbidx, index;
93	size_t offset = ctx->buf_offset;
94	ssize_t pagecpy;
95	struct lib_ring_buffer_backend_pages *rpages;
96	unsigned long sb_bindex, id;
97
98	offset &= chanb->buf_size - 1;
99	sbidx = offset >> chanb->subbuf_size_order;
100	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
101	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
102	id = bufb->buf_wsb[sbidx].id;
103	sb_bindex = subbuffer_id_get_index(config, id);
104	rpages = bufb->array[sb_bindex];
105	CHAN_WARN_ON(ctx->chan,
106	config->mode == RING_BUFFER_OVERWRITE
107	&& subbuffer_id_is_noref(config, id));
108	if (likely(pagecpy == len))
109	lib_ring_buffer_do_copy(config,
110	rpages->p[index].virt
111	+ (offset & ~PAGE_MASK),
112	src, len);
113	else
114	_lib_ring_buffer_write(bufb, offset, src, len, 0);
115	ctx->buf_offset += len;
116	}
117
4ea00e4f JD	118	/**
	119	* lib_ring_buffer_memset - write len bytes of c to a buffer backend
	120	* @config : ring buffer instance configuration
	121	* @bufb : ring buffer backend
	122	* @offset : offset within the buffer
	123	* @c : the byte to copy
	124	* @len : number of bytes to copy
	125	*
	126	* This function writes "len" bytes of "c" to a buffer backend, at a specific
	127	* offset. This is more or less a buffer backend-specific memset() operation.
	128	* Calls the slow path (_ring_buffer_memset) if write is crossing a page
	129	* boundary.
	130	*/
	131	static inline
	132	void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config,
	133	struct lib_ring_buffer_ctx *ctx, int c, size_t len)
	134	{
	135
	136	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	137	struct channel_backend *chanb = &ctx->chan->backend;
	138	size_t sbidx, index;
	139	size_t offset = ctx->buf_offset;
	140	ssize_t pagecpy;
	141	struct lib_ring_buffer_backend_pages *rpages;
	142	unsigned long sb_bindex, id;
	143
	144	offset &= chanb->buf_size - 1;
	145	sbidx = offset >> chanb->subbuf_size_order;
	146	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	147	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	148	id = bufb->buf_wsb[sbidx].id;
	149	sb_bindex = subbuffer_id_get_index(config, id);
	150	rpages = bufb->array[sb_bindex];
	151	CHAN_WARN_ON(ctx->chan,
	152	config->mode == RING_BUFFER_OVERWRITE
	153	&& subbuffer_id_is_noref(config, id));
	154	if (likely(pagecpy == len))
	155	lib_ring_buffer_do_memset(rpages->p[index].virt
	156	+ (offset & ~PAGE_MASK),
	157	c, len);
	158	else
	159	_lib_ring_buffer_memset(bufb, offset, c, len, 0);
	160	ctx->buf_offset += len;
	161	}
	162
	163	/**
	164	* lib_ring_buffer_copy_from_user - write userspace data to a buffer backend
	165	* @config : ring buffer instance configuration
	166	* @ctx: ring buffer context. (input arguments only)
	167	* @src : userspace source pointer to copy from
	168	* @len : length of data to copy
	169	*
	170	* This function copies "len" bytes of data from a userspace pointer to a
	171	* buffer backend, at the current context offset. This is more or less a buffer
	172	* backend-specific memcpy() operation. Calls the slow path
	173	* (_ring_buffer_write_from_user) if copy is crossing a page boundary.
	174	*/
	175	static inline
	176	void lib_ring_buffer_copy_from_user(const struct lib_ring_buffer_config *config,
	177	struct lib_ring_buffer_ctx *ctx,
	178	const void __user *src, size_t len)
	179	{
	180	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	181	struct channel_backend *chanb = &ctx->chan->backend;
182	size_t sbidx, index;
183	size_t offset = ctx->buf_offset;
184	ssize_t pagecpy;
185	struct lib_ring_buffer_backend_pages *rpages;
186	unsigned long sb_bindex, id;
187	unsigned long ret;
188
189	offset &= chanb->buf_size - 1;
190	sbidx = offset >> chanb->subbuf_size_order;
191	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
192	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
193	id = bufb->buf_wsb[sbidx].id;
194	sb_bindex = subbuffer_id_get_index(config, id);
195	rpages = bufb->array[sb_bindex];
196	CHAN_WARN_ON(ctx->chan,
197	config->mode == RING_BUFFER_OVERWRITE
198	&& subbuffer_id_is_noref(config, id));
199
200	if (unlikely(!access_ok(VERIFY_READ, src, len)))
201	goto fill_buffer;
202
203	if (likely(pagecpy == len)) {
204	ret = lib_ring_buffer_do_copy_from_user(
205	rpages->p[index].virt + (offset & ~PAGE_MASK),
206	src, len);
207	if (unlikely(ret > 0)) {
208	len -= (pagecpy - ret);
209	offset += (pagecpy - ret);
210	goto fill_buffer;
211	}
212	} else {
213	_lib_ring_buffer_copy_from_user(bufb, offset, src, len, 0);
214	}
215	ctx->buf_offset += len;
216
217	return;
218
219	fill_buffer:
220	/*
221	* In the error path we call the slow path version to avoid
222	* the pollution of static inline code.
223	*/
224	_lib_ring_buffer_memset(bufb, offset, 0, len, 0);
225	}
226
f3bc08c5 MD	227	/*
	228	* This accessor counts the number of unread records in a buffer.
	229	* It only provides a consistent value if no reads not writes are performed
	230	* concurrently.
	231	*/
	232	static inline
	233	unsigned long lib_ring_buffer_get_records_unread(
	234	const struct lib_ring_buffer_config *config,
	235	struct lib_ring_buffer *buf)
	236	{
	237	struct lib_ring_buffer_backend *bufb = &buf->backend;
	238	struct lib_ring_buffer_backend_pages *pages;
	239	unsigned long records_unread = 0, sb_bindex, id;
	240	unsigned int i;
	241
	242	for (i = 0; i < bufb->chan->backend.num_subbuf; i++) {
	243	id = bufb->buf_wsb[i].id;
	244	sb_bindex = subbuffer_id_get_index(config, id);
	245	pages = bufb->array[sb_bindex];
	246	records_unread += v_read(config, &pages->records_unread);
	247	}
	248	if (config->mode == RING_BUFFER_OVERWRITE) {
	249	id = bufb->buf_rsb.id;
	250	sb_bindex = subbuffer_id_get_index(config, id);
	251	pages = bufb->array[sb_bindex];
	252	records_unread += v_read(config, &pages->records_unread);
	253	}
	254	return records_unread;
	255	}
	256
	257	ssize_t lib_ring_buffer_file_splice_read(struct file in, loff_t ppos,
	258	struct pipe_inode_info *pipe,
	259	size_t len, unsigned int flags);
	260	loff_t lib_ring_buffer_no_llseek(struct file *file, loff_t offset, int origin);
	261
886d51a3	262	#endif /* _LIB_RING_BUFFER_BACKEND_H */