1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License as published by
4 * the Free Software Foundation; either version 2 of the License, or
5 * (at your option) any later version.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software
14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
15 *
16 * Copyright (C) IBM Corporation, 2012
17 *
18 * Author: Anton Blanchard <anton@au.ibm.com>
19 */
20
21#include <asm/ppc_asm.h>
22#include <asm/asm-offsets.h>
23
24	.section	".toc","aw"
25PPC64_CACHES:
26	.tc		ppc64_caches[TC],ppc64_caches
27	.section	".text"
28
29/**
30 * __clear_user: - Zero a block of memory in user space, with less checking.
31 * @to:   Destination address, in user space.
32 * @n:    Number of bytes to zero.
33 *
34 * Zero a block of memory in user space.  Caller must check
35 * the specified block with access_ok() before calling this function.
36 *
37 * Returns number of bytes that could not be cleared.
38 * On success, this will be zero.
39 */
40
41	.macro err1
42100:
43	.section __ex_table,"a"
44	.align 3
45	.llong 100b,.Ldo_err1
46	.previous
47	.endm
48
49	.macro err2
50200:
51	.section __ex_table,"a"
52	.align 3
53	.llong 200b,.Ldo_err2
54	.previous
55	.endm
56
57	.macro err3
58300:
59	.section __ex_table,"a"
60	.align 3
61	.llong 300b,.Ldo_err3
62	.previous
63	.endm
64
65.Ldo_err1:
66	mr	r3,r8
67
68.Ldo_err2:
69	mtctr	r4
701:
71err3;	stb	r0,0(r3)
72	addi	r3,r3,1
73	addi	r4,r4,-1
74	bdnz	1b
75
76.Ldo_err3:
77	mr	r3,r4
78	blr
79
80_GLOBAL_TOC(__clear_user)
81	cmpdi	r4,32
82	neg	r6,r3
83	li	r0,0
84	blt	.Lshort_clear
85	mr	r8,r3
86	mtocrf	0x01,r6
87	clrldi	r6,r6,(64-3)
88
89	/* Get the destination 8 byte aligned */
90	bf	cr7*4+3,1f
91err1;	stb	r0,0(r3)
92	addi	r3,r3,1
93
941:	bf	cr7*4+2,2f
95err1;	sth	r0,0(r3)
96	addi	r3,r3,2
97
982:	bf	cr7*4+1,3f
99err1;	stw	r0,0(r3)
100	addi	r3,r3,4
101
1023:	sub	r4,r4,r6
103
104	cmpdi	r4,32
105	cmpdi	cr1,r4,512
106	blt	.Lshort_clear
107	bgt	cr1,.Llong_clear
108
109.Lmedium_clear:
110	srdi	r6,r4,5
111	mtctr	r6
112
113	/* Do 32 byte chunks */
1144:
115err2;	std	r0,0(r3)
116err2;	std	r0,8(r3)
117err2;	std	r0,16(r3)
118err2;	std	r0,24(r3)
119	addi	r3,r3,32
120	addi	r4,r4,-32
121	bdnz	4b
122
123.Lshort_clear:
124	/* up to 31 bytes to go */
125	cmpdi	r4,16
126	blt	6f
127err2;	std	r0,0(r3)
128err2;	std	r0,8(r3)
129	addi	r3,r3,16
130	addi	r4,r4,-16
131
132	/* Up to 15 bytes to go */
1336:	mr	r8,r3
134	clrldi	r4,r4,(64-4)
135	mtocrf	0x01,r4
136	bf	cr7*4+0,7f
137err1;	std	r0,0(r3)
138	addi	r3,r3,8
139
1407:	bf	cr7*4+1,8f
141err1;	stw	r0,0(r3)
142	addi	r3,r3,4
143
1448:	bf	cr7*4+2,9f
145err1;	sth	r0,0(r3)
146	addi	r3,r3,2
147
1489:	bf	cr7*4+3,10f
149err1;	stb	r0,0(r3)
150
15110:	li	r3,0
152	blr
153
154.Llong_clear:
155	ld	r5,PPC64_CACHES@toc(r2)
156
157	bf	cr7*4+0,11f
158err2;	std	r0,0(r3)
159	addi	r3,r3,8
160	addi	r4,r4,-8
161
162	/* Destination is 16 byte aligned, need to get it cacheline aligned */
16311:	lwz	r7,DCACHEL1LOGLINESIZE(r5)
164	lwz	r9,DCACHEL1LINESIZE(r5)
165
166	/*
167	 * With worst case alignment the long clear loop takes a minimum
168	 * of 1 byte less than 2 cachelines.
169	 */
170	sldi	r10,r9,2
171	cmpd	r4,r10
172	blt	.Lmedium_clear
173
174	neg	r6,r3
175	addi	r10,r9,-1
176	and.	r5,r6,r10
177	beq	13f
178
179	srdi	r6,r5,4
180	mtctr	r6
181	mr	r8,r3
18212:
183err1;	std	r0,0(r3)
184err1;	std	r0,8(r3)
185	addi	r3,r3,16
186	bdnz	12b
187
188	sub	r4,r4,r5
189
19013:	srd	r6,r4,r7
191	mtctr	r6
192	mr	r8,r3
19314:
194err1;	dcbz	r0,r3
195	add	r3,r3,r9
196	bdnz	14b
197
198	and	r4,r4,r10
199
200	cmpdi	r4,32
201	blt	.Lshort_clear
202	b	.Lmedium_clear
203