1#ifndef _ASM_X86_UACCESS_64_H
2#define _ASM_X86_UACCESS_64_H
3
4/*
5 * User space memory access functions
6 */
7#include <linux/compiler.h>
8#include <linux/errno.h>
9#include <linux/lockdep.h>
10#include <asm/alternative.h>
11#include <asm/cpufeature.h>
12#include <asm/page.h>
13
14/*
15 * Copy To/From Userspace
16 */
17
18/* Handles exceptions in both to and from, but doesn't do access_ok */
19__must_check unsigned long
20copy_user_enhanced_fast_string(void *to, const void *from, unsigned len);
21__must_check unsigned long
22copy_user_generic_string(void *to, const void *from, unsigned len);
23__must_check unsigned long
24copy_user_generic_unrolled(void *to, const void *from, unsigned len);
25
26static __always_inline __must_check unsigned long
27copy_user_generic(void *to, const void *from, unsigned len)
28{
29	unsigned ret;
30
31	/*
32	 * If CPU has ERMS feature, use copy_user_enhanced_fast_string.
33	 * Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
34	 * Otherwise, use copy_user_generic_unrolled.
35	 */
36	alternative_call_2(copy_user_generic_unrolled,
37			 copy_user_generic_string,
38			 X86_FEATURE_REP_GOOD,
39			 copy_user_enhanced_fast_string,
40			 X86_FEATURE_ERMS,
41			 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
42				     "=d" (len)),
43			 "1" (to), "2" (from), "3" (len)
44			 : "memory", "rcx", "r8", "r9", "r10", "r11");
45	return ret;
46}
47
48__must_check unsigned long
49copy_in_user(void __user *to, const void __user *from, unsigned len);
50
51static __always_inline __must_check
52int __copy_from_user_nocheck(void *dst, const void __user *src, unsigned size)
53{
54	int ret = 0;
55
56	if (!__builtin_constant_p(size))
57		return copy_user_generic(dst, (__force void *)src, size);
58	switch (size) {
59	case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
60			      ret, "b", "b", "=q", 1);
61		return ret;
62	case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
63			      ret, "w", "w", "=r", 2);
64		return ret;
65	case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
66			      ret, "l", "k", "=r", 4);
67		return ret;
68	case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
69			      ret, "q", "", "=r", 8);
70		return ret;
71	case 10:
72		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
73			       ret, "q", "", "=r", 10);
74		if (unlikely(ret))
75			return ret;
76		__get_user_asm(*(u16 *)(8 + (char *)dst),
77			       (u16 __user *)(8 + (char __user *)src),
78			       ret, "w", "w", "=r", 2);
79		return ret;
80	case 16:
81		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
82			       ret, "q", "", "=r", 16);
83		if (unlikely(ret))
84			return ret;
85		__get_user_asm(*(u64 *)(8 + (char *)dst),
86			       (u64 __user *)(8 + (char __user *)src),
87			       ret, "q", "", "=r", 8);
88		return ret;
89	default:
90		return copy_user_generic(dst, (__force void *)src, size);
91	}
92}
93
94static __always_inline __must_check
95int __copy_from_user(void *dst, const void __user *src, unsigned size)
96{
97	might_fault();
98	return __copy_from_user_nocheck(dst, src, size);
99}
100
101static __always_inline __must_check
102int __copy_to_user_nocheck(void __user *dst, const void *src, unsigned size)
103{
104	int ret = 0;
105
106	if (!__builtin_constant_p(size))
107		return copy_user_generic((__force void *)dst, src, size);
108	switch (size) {
109	case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
110			      ret, "b", "b", "iq", 1);
111		return ret;
112	case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
113			      ret, "w", "w", "ir", 2);
114		return ret;
115	case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
116			      ret, "l", "k", "ir", 4);
117		return ret;
118	case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
119			      ret, "q", "", "er", 8);
120		return ret;
121	case 10:
122		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
123			       ret, "q", "", "er", 10);
124		if (unlikely(ret))
125			return ret;
126		asm("":::"memory");
127		__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
128			       ret, "w", "w", "ir", 2);
129		return ret;
130	case 16:
131		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
132			       ret, "q", "", "er", 16);
133		if (unlikely(ret))
134			return ret;
135		asm("":::"memory");
136		__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
137			       ret, "q", "", "er", 8);
138		return ret;
139	default:
140		return copy_user_generic((__force void *)dst, src, size);
141	}
142}
143
144static __always_inline __must_check
145int __copy_to_user(void __user *dst, const void *src, unsigned size)
146{
147	might_fault();
148	return __copy_to_user_nocheck(dst, src, size);
149}
150
151static __always_inline __must_check
152int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
153{
154	int ret = 0;
155
156	might_fault();
157	if (!__builtin_constant_p(size))
158		return copy_user_generic((__force void *)dst,
159					 (__force void *)src, size);
160	switch (size) {
161	case 1: {
162		u8 tmp;
163		__get_user_asm(tmp, (u8 __user *)src,
164			       ret, "b", "b", "=q", 1);
165		if (likely(!ret))
166			__put_user_asm(tmp, (u8 __user *)dst,
167				       ret, "b", "b", "iq", 1);
168		return ret;
169	}
170	case 2: {
171		u16 tmp;
172		__get_user_asm(tmp, (u16 __user *)src,
173			       ret, "w", "w", "=r", 2);
174		if (likely(!ret))
175			__put_user_asm(tmp, (u16 __user *)dst,
176				       ret, "w", "w", "ir", 2);
177		return ret;
178	}
179
180	case 4: {
181		u32 tmp;
182		__get_user_asm(tmp, (u32 __user *)src,
183			       ret, "l", "k", "=r", 4);
184		if (likely(!ret))
185			__put_user_asm(tmp, (u32 __user *)dst,
186				       ret, "l", "k", "ir", 4);
187		return ret;
188	}
189	case 8: {
190		u64 tmp;
191		__get_user_asm(tmp, (u64 __user *)src,
192			       ret, "q", "", "=r", 8);
193		if (likely(!ret))
194			__put_user_asm(tmp, (u64 __user *)dst,
195				       ret, "q", "", "er", 8);
196		return ret;
197	}
198	default:
199		return copy_user_generic((__force void *)dst,
200					 (__force void *)src, size);
201	}
202}
203
204static __must_check __always_inline int
205__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
206{
207	return __copy_from_user_nocheck(dst, src, size);
208}
209
210static __must_check __always_inline int
211__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
212{
213	return __copy_to_user_nocheck(dst, src, size);
214}
215
216extern long __copy_user_nocache(void *dst, const void __user *src,
217				unsigned size, int zerorest);
218
219static inline int
220__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
221{
222	might_fault();
223	return __copy_user_nocache(dst, src, size, 1);
224}
225
226static inline int
227__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
228				  unsigned size)
229{
230	return __copy_user_nocache(dst, src, size, 0);
231}
232
233unsigned long
234copy_user_handle_tail(char *to, char *from, unsigned len);
235
236#endif /* _ASM_X86_UACCESS_64_H */
237