Commit | Line | Data |
---|---|---|
b7cdc182 | 1 | /* SPDX-License-Identifier: GPL-2.0-only |
9f36eaed | 2 | * |
f3bc08c5 MD |
3 | * ring_buffer_splice.c |
4 | * | |
5 | * Copyright (C) 2002-2005 - Tom Zanussi <zanussi@us.ibm.com>, IBM Corp | |
6 | * Copyright (C) 1999-2005 - Karim Yaghmour <karim@opersys.com> | |
886d51a3 | 7 | * Copyright (C) 2008-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> |
f3bc08c5 | 8 | * |
886d51a3 | 9 | * Re-using code from kernel/relay.c, which is why it is licensed under |
9f36eaed | 10 | * the GPL-2.0. |
f3bc08c5 MD |
11 | */ |
12 | ||
13 | #include <linux/module.h> | |
14 | #include <linux/fs.h> | |
5f4c791e | 15 | #include <lttng/kernel-version.h> |
f3bc08c5 | 16 | |
c075712b | 17 | #include <wrapper/splice.h> |
24591303 MD |
18 | #include <ringbuffer/backend.h> |
19 | #include <ringbuffer/frontend.h> | |
20 | #include <ringbuffer/vfs.h> | |
f3bc08c5 | 21 | |
5a15f70c | 22 | #ifdef DEBUG |
f3bc08c5 MD |
23 | #define printk_dbg(fmt, args...) printk(fmt, args) |
24 | #else | |
5a15f70c MJ |
25 | #define printk_dbg(fmt, args...) \ |
26 | do { \ | |
27 | /* do nothing but check printf format */ \ | |
28 | if (0) \ | |
29 | printk(fmt, ## args); \ | |
30 | } while (0) | |
f3bc08c5 MD |
31 | #endif |
32 | ||
d83004aa JD |
33 | loff_t vfs_lib_ring_buffer_no_llseek(struct file *file, loff_t offset, |
34 | int origin) | |
f3bc08c5 MD |
35 | { |
36 | return -ESPIPE; | |
37 | } | |
d83004aa | 38 | EXPORT_SYMBOL_GPL(vfs_lib_ring_buffer_no_llseek); |
f3bc08c5 MD |
39 | |
40 | /* | |
41 | * Release pages from the buffer so splice pipe_to_file can move them. | |
42 | * Called after the pipe has been populated with buffer pages. | |
43 | */ | |
44 | static void lib_ring_buffer_pipe_buf_release(struct pipe_inode_info *pipe, | |
45 | struct pipe_buffer *pbuf) | |
46 | { | |
47 | __free_page(pbuf->page); | |
48 | } | |
49 | ||
5f4c791e | 50 | #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0)) |
4352d508 MJ |
51 | static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = { |
52 | .release = lib_ring_buffer_pipe_buf_release, | |
53 | .try_steal = generic_pipe_buf_try_steal, | |
54 | .get = generic_pipe_buf_get | |
55 | }; | |
5f4c791e | 56 | #elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,1,0)) |
4352d508 MJ |
57 | static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = { |
58 | .confirm = generic_pipe_buf_confirm, | |
59 | .release = lib_ring_buffer_pipe_buf_release, | |
60 | .steal = generic_pipe_buf_steal, | |
61 | .get = generic_pipe_buf_get | |
62 | }; | |
4352d508 | 63 | #else |
f3bc08c5 MD |
64 | static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = { |
65 | .can_merge = 0, | |
f3bc08c5 MD |
66 | .confirm = generic_pipe_buf_confirm, |
67 | .release = lib_ring_buffer_pipe_buf_release, | |
68 | .steal = generic_pipe_buf_steal, | |
4352d508 | 69 | .get = generic_pipe_buf_get |
f3bc08c5 | 70 | }; |
4352d508 | 71 | #endif |
f3bc08c5 MD |
72 | |
73 | /* | |
74 | * Page release operation after splice pipe_to_file ends. | |
75 | */ | |
76 | static void lib_ring_buffer_page_release(struct splice_pipe_desc *spd, | |
77 | unsigned int i) | |
78 | { | |
79 | __free_page(spd->pages[i]); | |
80 | } | |
81 | ||
82 | /* | |
83 | * subbuf_splice_actor - splice up to one subbuf's worth of data | |
84 | */ | |
85 | static int subbuf_splice_actor(struct file *in, | |
86 | loff_t *ppos, | |
87 | struct pipe_inode_info *pipe, | |
88 | size_t len, | |
d83004aa | 89 | unsigned int flags, |
e20c0fec | 90 | struct lttng_kernel_ring_buffer *buf) |
f3bc08c5 | 91 | { |
860c213b | 92 | struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan; |
e20c0fec | 93 | const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
94 | unsigned int poff, subbuf_pages, nr_pages; |
95 | struct page *pages[PIPE_DEF_BUFFERS]; | |
96 | struct partial_page partial[PIPE_DEF_BUFFERS]; | |
97 | struct splice_pipe_desc spd = { | |
98 | .pages = pages, | |
99 | .nr_pages = 0, | |
100 | .partial = partial, | |
5f4c791e | 101 | #if (LTTNG_LINUX_VERSION_CODE < LTTNG_KERNEL_VERSION(4,12,0)) |
f3bc08c5 | 102 | .flags = flags, |
e2833bed | 103 | #endif |
f3bc08c5 MD |
104 | .ops = &ring_buffer_pipe_buf_ops, |
105 | .spd_release = lib_ring_buffer_page_release, | |
106 | }; | |
88dfd899 | 107 | unsigned long consumed_old, roffset; |
f3bc08c5 MD |
108 | unsigned long bytes_avail; |
109 | ||
110 | /* | |
111 | * Check that a GET_SUBBUF ioctl has been done before. | |
112 | */ | |
113 | WARN_ON(atomic_long_read(&buf->active_readers) != 1); | |
114 | consumed_old = lib_ring_buffer_get_consumed(config, buf); | |
115 | consumed_old += *ppos; | |
f3bc08c5 MD |
116 | |
117 | /* | |
118 | * Adjust read len, if longer than what is available. | |
119 | * Max read size is 1 subbuffer due to get_subbuf/put_subbuf for | |
120 | * protection. | |
121 | */ | |
122 | bytes_avail = chan->backend.subbuf_size; | |
123 | WARN_ON(bytes_avail > chan->backend.buf_size); | |
124 | len = min_t(size_t, len, bytes_avail); | |
125 | subbuf_pages = bytes_avail >> PAGE_SHIFT; | |
126 | nr_pages = min_t(unsigned int, subbuf_pages, PIPE_DEF_BUFFERS); | |
127 | roffset = consumed_old & PAGE_MASK; | |
128 | poff = consumed_old & ~PAGE_MASK; | |
5a15f70c | 129 | printk_dbg(KERN_DEBUG "LTTng: SPLICE actor len %zu pos %zd write_pos %ld\n", |
f3bc08c5 MD |
130 | len, (ssize_t)*ppos, lib_ring_buffer_get_offset(config, buf)); |
131 | ||
132 | for (; spd.nr_pages < nr_pages; spd.nr_pages++) { | |
133 | unsigned int this_len; | |
0112cb7b MD |
134 | unsigned long *pfnp, new_pfn; |
135 | struct page *new_page; | |
f3bc08c5 MD |
136 | void **virt; |
137 | ||
138 | if (!len) | |
139 | break; | |
5a15f70c | 140 | printk_dbg(KERN_DEBUG "LTTng: SPLICE actor loop len %zu roffset %ld\n", |
f3bc08c5 MD |
141 | len, roffset); |
142 | ||
143 | /* | |
144 | * We have to replace the page we are moving into the splice | |
145 | * pipe. | |
146 | */ | |
147 | new_page = alloc_pages_node(cpu_to_node(max(buf->backend.cpu, | |
148 | 0)), | |
149 | GFP_KERNEL | __GFP_ZERO, 0); | |
150 | if (!new_page) | |
151 | break; | |
0112cb7b | 152 | new_pfn = page_to_pfn(new_page); |
f3bc08c5 | 153 | this_len = PAGE_SIZE - poff; |
0112cb7b MD |
154 | pfnp = lib_ring_buffer_read_get_pfn(&buf->backend, roffset, &virt); |
155 | spd.pages[spd.nr_pages] = pfn_to_page(*pfnp); | |
156 | *pfnp = new_pfn; | |
f3bc08c5 MD |
157 | *virt = page_address(new_page); |
158 | spd.partial[spd.nr_pages].offset = poff; | |
159 | spd.partial[spd.nr_pages].len = this_len; | |
160 | ||
161 | poff = 0; | |
162 | roffset += PAGE_SIZE; | |
163 | len -= this_len; | |
164 | } | |
165 | ||
166 | if (!spd.nr_pages) | |
167 | return 0; | |
168 | ||
90225db5 | 169 | return wrapper_splice_to_pipe(pipe, &spd); |
f3bc08c5 MD |
170 | } |
171 | ||
172 | ssize_t lib_ring_buffer_splice_read(struct file *in, loff_t *ppos, | |
173 | struct pipe_inode_info *pipe, size_t len, | |
d83004aa | 174 | unsigned int flags, |
e20c0fec | 175 | struct lttng_kernel_ring_buffer *buf) |
f3bc08c5 | 176 | { |
860c213b | 177 | struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan; |
e20c0fec | 178 | const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config; |
f3bc08c5 MD |
179 | ssize_t spliced; |
180 | int ret; | |
181 | ||
182 | if (config->output != RING_BUFFER_SPLICE) | |
183 | return -EINVAL; | |
184 | ||
185 | /* | |
186 | * We require ppos and length to be page-aligned for performance reasons | |
187 | * (no page copy). Size is known using the ioctl | |
188 | * RING_BUFFER_GET_PADDED_SUBBUF_SIZE, which is page-size padded. | |
189 | * We fail when the ppos or len passed is not page-sized, because splice | |
190 | * is not allowed to copy more than the length passed as parameter (so | |
191 | * the ABI does not let us silently copy more than requested to include | |
192 | * padding). | |
193 | */ | |
194 | if (*ppos != PAGE_ALIGN(*ppos) || len != PAGE_ALIGN(len)) | |
195 | return -EINVAL; | |
196 | ||
197 | ret = 0; | |
198 | spliced = 0; | |
199 | ||
5a15f70c | 200 | printk_dbg(KERN_DEBUG "LTTng: SPLICE read len %zu pos %zd\n", len, |
f3bc08c5 MD |
201 | (ssize_t)*ppos); |
202 | while (len && !spliced) { | |
d83004aa | 203 | ret = subbuf_splice_actor(in, ppos, pipe, len, flags, buf); |
5a15f70c | 204 | printk_dbg(KERN_DEBUG "LTTng: SPLICE read loop ret %d\n", ret); |
f3bc08c5 MD |
205 | if (ret < 0) |
206 | break; | |
207 | else if (!ret) { | |
208 | if (flags & SPLICE_F_NONBLOCK) | |
209 | ret = -EAGAIN; | |
210 | break; | |
211 | } | |
212 | ||
213 | *ppos += ret; | |
214 | if (ret > len) | |
215 | len = 0; | |
216 | else | |
217 | len -= ret; | |
218 | spliced += ret; | |
219 | } | |
220 | ||
221 | if (spliced) | |
222 | return spliced; | |
223 | ||
224 | return ret; | |
225 | } | |
226 | EXPORT_SYMBOL_GPL(lib_ring_buffer_splice_read); | |
d83004aa JD |
227 | |
228 | ssize_t vfs_lib_ring_buffer_splice_read(struct file *in, loff_t *ppos, | |
229 | struct pipe_inode_info *pipe, size_t len, | |
230 | unsigned int flags) | |
231 | { | |
e20c0fec | 232 | struct lttng_kernel_ring_buffer *buf = in->private_data; |
d83004aa JD |
233 | |
234 | return lib_ring_buffer_splice_read(in, ppos, pipe, len, flags, buf); | |
235 | } | |
236 | EXPORT_SYMBOL_GPL(vfs_lib_ring_buffer_splice_read); |