1/*
2 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
3 * Copyright 2002 Andi Kleen, SuSE Labs.
4 * Subject to the GNU Public License v2.
5 *
6 * Functions to copy from and to user space.
7 */
8
9#include <linux/linkage.h>
10#include <asm/dwarf2.h>
11
12#define FIX_ALIGNMENT 1
13
14#include <asm/current.h>
15#include <asm/asm-offsets.h>
16#include <asm/thread_info.h>
17#include <asm/asm.h>
18#include <asm/smap.h>
19
20	.macro ALIGN_DESTINATION
21#ifdef FIX_ALIGNMENT
22	/* check for bad alignment of destination */
23	movl %edi,%ecx
24	andl $7,%ecx
25	jz 102f				/* already aligned */
26	subl $8,%ecx
27	negl %ecx
28	subl %ecx,%edx
29100:	movb (%rsi),%al
30101:	movb %al,(%rdi)
31	incq %rsi
32	incq %rdi
33	decl %ecx
34	jnz 100b
35102:
36	.section .fixup,"ax"
37103:	addl %ecx,%edx			/* ecx is zerorest also */
38	jmp copy_user_handle_tail
39	.previous
40
41	_ASM_EXTABLE(100b,103b)
42	_ASM_EXTABLE(101b,103b)
43#endif
44	.endm
45
46/*
47 * copy_user_nocache - Uncached memory copy with exception handling
48 * This will force destination/source out of cache for more performance.
49 */
50ENTRY(__copy_user_nocache)
51	CFI_STARTPROC
52	ASM_STAC
53	cmpl $8,%edx
54	jb 20f		/* less then 8 bytes, go to byte copy loop */
55	ALIGN_DESTINATION
56	movl %edx,%ecx
57	andl $63,%edx
58	shrl $6,%ecx
59	jz 17f
601:	movq (%rsi),%r8
612:	movq 1*8(%rsi),%r9
623:	movq 2*8(%rsi),%r10
634:	movq 3*8(%rsi),%r11
645:	movnti %r8,(%rdi)
656:	movnti %r9,1*8(%rdi)
667:	movnti %r10,2*8(%rdi)
678:	movnti %r11,3*8(%rdi)
689:	movq 4*8(%rsi),%r8
6910:	movq 5*8(%rsi),%r9
7011:	movq 6*8(%rsi),%r10
7112:	movq 7*8(%rsi),%r11
7213:	movnti %r8,4*8(%rdi)
7314:	movnti %r9,5*8(%rdi)
7415:	movnti %r10,6*8(%rdi)
7516:	movnti %r11,7*8(%rdi)
76	leaq 64(%rsi),%rsi
77	leaq 64(%rdi),%rdi
78	decl %ecx
79	jnz 1b
8017:	movl %edx,%ecx
81	andl $7,%edx
82	shrl $3,%ecx
83	jz 20f
8418:	movq (%rsi),%r8
8519:	movnti %r8,(%rdi)
86	leaq 8(%rsi),%rsi
87	leaq 8(%rdi),%rdi
88	decl %ecx
89	jnz 18b
9020:	andl %edx,%edx
91	jz 23f
92	movl %edx,%ecx
9321:	movb (%rsi),%al
9422:	movb %al,(%rdi)
95	incq %rsi
96	incq %rdi
97	decl %ecx
98	jnz 21b
9923:	xorl %eax,%eax
100	ASM_CLAC
101	sfence
102	ret
103
104	.section .fixup,"ax"
10530:	shll $6,%ecx
106	addl %ecx,%edx
107	jmp 60f
10840:	lea (%rdx,%rcx,8),%rdx
109	jmp 60f
11050:	movl %ecx,%edx
11160:	sfence
112	jmp copy_user_handle_tail
113	.previous
114
115	_ASM_EXTABLE(1b,30b)
116	_ASM_EXTABLE(2b,30b)
117	_ASM_EXTABLE(3b,30b)
118	_ASM_EXTABLE(4b,30b)
119	_ASM_EXTABLE(5b,30b)
120	_ASM_EXTABLE(6b,30b)
121	_ASM_EXTABLE(7b,30b)
122	_ASM_EXTABLE(8b,30b)
123	_ASM_EXTABLE(9b,30b)
124	_ASM_EXTABLE(10b,30b)
125	_ASM_EXTABLE(11b,30b)
126	_ASM_EXTABLE(12b,30b)
127	_ASM_EXTABLE(13b,30b)
128	_ASM_EXTABLE(14b,30b)
129	_ASM_EXTABLE(15b,30b)
130	_ASM_EXTABLE(16b,30b)
131	_ASM_EXTABLE(18b,40b)
132	_ASM_EXTABLE(19b,40b)
133	_ASM_EXTABLE(21b,50b)
134	_ASM_EXTABLE(22b,50b)
135	CFI_ENDPROC
136ENDPROC(__copy_user_nocache)
137