1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Access kernel memory without faulting.
4 */
5 #include <linux/export.h>
6 #include <linux/mm.h>
7 #include <linux/uaccess.h>
8
9 static __always_inline long
10 probe_read_common(void *dst, const void __user *src, size_t size)
11 {
12 long ret;
13
14 pagefault_disable();
15 ret = __copy_from_user_inatomic(dst, src, size);
16 pagefault_enable();
17
18 return ret ? -EFAULT : 0;
19 }
20
21 static __always_inline long
22 probe_write_common(void __user *dst, const void *src, size_t size)
23 {
24 long ret;
25
26 pagefault_disable();
27 ret = __copy_to_user_inatomic(dst, src, size);
28 pagefault_enable();
29
30 return ret ? -EFAULT : 0;
31 }
32
33 /**
34 * probe_kernel_read(): safely attempt to read from a kernel-space location
35 * @dst: pointer to the buffer that shall take the data
36 * @src: address to read from
37 * @size: size of the data chunk
38 *
39 * Safely read from address @src to the buffer at @dst. If a kernel fault
40 * happens, handle that and return -EFAULT.
41 *
42 * We ensure that the copy_from_user is executed in atomic context so that
43 * do_page_fault() doesn't attempt to take mmap_sem. This makes
44 * probe_kernel_read() suitable for use within regions where the caller
45 * already holds mmap_sem, or other locks which nest inside mmap_sem.
46 */
47
48 long __weak probe_kernel_read(void *dst, const void *src, size_t size)
49 __attribute__((alias("__probe_kernel_read")));
50
51 long __probe_kernel_read(void *dst, const void *src, size_t size)
52 {
53 long ret;
54 mm_segment_t old_fs = get_fs();
55
56 set_fs(KERNEL_DS);
57 ret = probe_read_common(dst, (__force const void __user *)src, size);
58 set_fs(old_fs);
59
60 return ret;
61 }
62 EXPORT_SYMBOL_GPL(probe_kernel_read);
63
64 /**
65 * probe_user_read(): safely attempt to read from a user-space location
66 * @dst: pointer to the buffer that shall take the data
67 * @src: address to read from. This must be a user address.
68 * @size: size of the data chunk
69 *
70 * Safely read from user address @src to the buffer at @dst. If a kernel fault
71 * happens, handle that and return -EFAULT.
72 */
73
74 long __weak probe_user_read(void *dst, const void __user *src, size_t size)
75 __attribute__((alias("__probe_user_read")));
76
77 long __probe_user_read(void *dst, const void __user *src, size_t size)
78 {
79 long ret = -EFAULT;
80 mm_segment_t old_fs = get_fs();
81
82 set_fs(USER_DS);
83 if (access_ok(src, size))
84 ret = probe_read_common(dst, src, size);
85 set_fs(old_fs);
86
87 return ret;
88 }
89 EXPORT_SYMBOL_GPL(probe_user_read);
90
91 /**
92 * probe_kernel_write(): safely attempt to write to a location
93 * @dst: address to write to
94 * @src: pointer to the data that shall be written
95 * @size: size of the data chunk
96 *
97 * Safely write to address @dst from the buffer at @src. If a kernel fault
98 * happens, handle that and return -EFAULT.
99 */
100
101 long __weak probe_kernel_write(void *dst, const void *src, size_t size)
102 __attribute__((alias("__probe_kernel_write")));
103
104 long __probe_kernel_write(void *dst, const void *src, size_t size)
105 {
106 long ret;
107 mm_segment_t old_fs = get_fs();
108
109 set_fs(KERNEL_DS);
110 ret = probe_write_common((__force void __user *)dst, src, size);
111 set_fs(old_fs);
112
113 return ret;
114 }
115 EXPORT_SYMBOL_GPL(probe_kernel_write);
116
117 /**
118 * probe_user_write(): safely attempt to write to a user-space location
119 * @dst: address to write to
120 * @src: pointer to the data that shall be written
121 * @size: size of the data chunk
122 *
123 * Safely write to address @dst from the buffer at @src. If a kernel fault
124 * happens, handle that and return -EFAULT.
125 */
126
127 long __weak probe_user_write(void __user *dst, const void *src, size_t size)
128 __attribute__((alias("__probe_user_write")));
129
130 long __probe_user_write(void __user *dst, const void *src, size_t size)
131 {
132 long ret = -EFAULT;
133 mm_segment_t old_fs = get_fs();
134
135 set_fs(USER_DS);
136 if (access_ok(dst, size))
137 ret = probe_write_common(dst, src, size);
138 set_fs(old_fs);
139
140 return ret;
141 }
142 EXPORT_SYMBOL_GPL(probe_user_write);
143
144 /**
145 * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address.
146 * @dst: Destination address, in kernel space. This buffer must be at
147 * least @count bytes long.
148 * @unsafe_addr: Unsafe address.
149 * @count: Maximum number of bytes to copy, including the trailing NUL.
150 *
151 * Copies a NUL-terminated string from unsafe address to kernel buffer.
152 *
153 * On success, returns the length of the string INCLUDING the trailing NUL.
154 *
155 * If access fails, returns -EFAULT (some data may have been copied
156 * and the trailing NUL added).
157 *
158 * If @count is smaller than the length of the string, copies @count-1 bytes,
159 * sets the last byte of @dst buffer to NUL and returns @count.
160 */
161 long strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count)
162 {
163 mm_segment_t old_fs = get_fs();
164 const void *src = unsafe_addr;
165 long ret;
166
167 if (unlikely(count <= 0))
168 return 0;
169
170 set_fs(KERNEL_DS);
171 pagefault_disable();
172
173 do {
174 ret = __get_user(*dst++, (const char __user __force *)src++);
175 } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
176
177 dst[-1] = '\0';
178 pagefault_enable();
179 set_fs(old_fs);
180
181 return ret ? -EFAULT : src - unsafe_addr;
182 }
183
184 /**
185 * strncpy_from_unsafe_user: - Copy a NUL terminated string from unsafe user
186 * address.
187 * @dst: Destination address, in kernel space. This buffer must be at
188 * least @count bytes long.
189 * @unsafe_addr: Unsafe user address.
190 * @count: Maximum number of bytes to copy, including the trailing NUL.
191 *
192 * Copies a NUL-terminated string from unsafe user address to kernel buffer.
193 *
194 * On success, returns the length of the string INCLUDING the trailing NUL.
195 *
196 * If access fails, returns -EFAULT (some data may have been copied
197 * and the trailing NUL added).
198 *
199 * If @count is smaller than the length of the string, copies @count-1 bytes,
200 * sets the last byte of @dst buffer to NUL and returns @count.
201 */
202 long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr,
203 long count)
204 {
205 mm_segment_t old_fs = get_fs();
206 long ret;
207
208 if (unlikely(count <= 0))
209 return 0;
210
211 set_fs(USER_DS);
212 pagefault_disable();
213 ret = strncpy_from_user(dst, unsafe_addr, count);
214 pagefault_enable();
215 set_fs(old_fs);
216
217 if (ret >= count) {
218 ret = count;
219 dst[ret - 1] = '\0';
220 } else if (ret > 0) {
221 ret++;
222 }
223
224 return ret;
225 }
226
227 /**
228 * strnlen_unsafe_user: - Get the size of a user string INCLUDING final NUL.
229 * @unsafe_addr: The string to measure.
230 * @count: Maximum count (including NUL)
231 *
232 * Get the size of a NUL-terminated string in user space without pagefault.
233 *
234 * Returns the size of the string INCLUDING the terminating NUL.
235 *
236 * If the string is too long, returns a number larger than @count. User
237 * has to check the return value against "> count".
238 * On exception (or invalid count), returns 0.
239 *
240 * Unlike strnlen_user, this can be used from IRQ handler etc. because
241 * it disables pagefaults.
242 */
243 long strnlen_unsafe_user(const void __user *unsafe_addr, long count)
244 {
245 mm_segment_t old_fs = get_fs();
246 int ret;
247
248 set_fs(USER_DS);
249 pagefault_disable();
250 ret = strnlen_user(unsafe_addr, count);
251 pagefault_enable();
252 set_fs(old_fs);
253
254 return ret;
255 }