1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * linux/arch/arm/lib/copy_template.s
4 *
5 * Code template for optimized memory copy functions
6 *
7 * Author: Nicolas Pitre
8 * Created: Sep 28, 2005
9 * Copyright: MontaVista Software, Inc.
10 */
11
12 /*
13 * Theory of operation
14 * -------------------
15 *
16 * This file provides the core code for a forward memory copy used in
17 * the implementation of memcopy(), copy_to_user() and copy_from_user().
18 *
19 * The including file must define the following accessor macros
20 * according to the need of the given function:
21 *
22 * ldr1w ptr reg abort
23 *
24 * This loads one word from 'ptr', stores it in 'reg' and increments
25 * 'ptr' to the next word. The 'abort' argument is used for fixup tables.
26 *
27 * ldr4w ptr reg1 reg2 reg3 reg4 abort
28 * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
29 *
30 * This loads four or eight words starting from 'ptr', stores them
31 * in provided registers and increments 'ptr' past those words.
32 * The'abort' argument is used for fixup tables.
33 *
34 * ldr1b ptr reg cond abort
35 *
36 * Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
37 * It also must apply the condition code if provided, otherwise the
38 * "al" condition is assumed by default.
39 *
40 * str1w ptr reg abort
41 * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
42 * str1b ptr reg cond abort
43 *
44 * Same as their ldr* counterparts, but data is stored to 'ptr' location
45 * rather than being loaded.
46 *
47 * enter reg1 reg2
48 *
49 * Preserve the provided registers on the stack plus any additional
50 * data as needed by the implementation including this code. Called
51 * upon code entry.
52 *
53 * usave reg1 reg2
54 *
55 * Unwind annotation macro is corresponding for 'enter' macro.
56 * It tell unwinder that preserved some provided registers on the stack
57 * and additional data by a prior 'enter' macro.
58 *
59 * exit reg1 reg2
60 *
61 * Restore registers with the values previously saved with the
62 * 'preserv' macro. Called upon code termination.
63 *
64 * LDR1W_SHIFT
65 * STR1W_SHIFT
66 *
67 * Correction to be applied to the "ip" register when branching into
68 * the ldr1w or str1w instructions (some of these macros may expand to
69 * than one 32bit instruction in Thumb-2)
70 */
71
72
73 UNWIND( .fnstart )
74 enter r4, lr
75 UNWIND( .fnend )
76
77 UNWIND( .fnstart )
78 usave r4, lr @ in first stmdb block
79
80 subs r2, r2, #4
81 blt 8f
82 ands ip, r0, #3
83 PLD( pld [r1, #0] )
84 bne 9f
85 ands ip, r1, #3
86 bne 10f
87
88 1: subs r2, r2, #(28)
89 stmfd sp!, {r5 - r8}
90 UNWIND( .fnend )
91
92 UNWIND( .fnstart )
93 usave r4, lr
94 UNWIND( .save {r5 - r8} ) @ in second stmfd block
95 blt 5f
96
97 CALGN( ands ip, r0, #31 )
98 CALGN( rsb r3, ip, #32 )
99 CALGN( sbcsne r4, r3, r2 ) @ C is always set here
100 CALGN( bcs 2f )
101 CALGN( adr r4, 6f )
102 CALGN( subs r2, r2, r3 ) @ C gets set
103 CALGN( add pc, r4, ip )
104
105 PLD( pld [r1, #0] )
106 2: PLD( subs r2, r2, #96 )
107 PLD( pld [r1, #28] )
108 PLD( blt 4f )
109 PLD( pld [r1, #60] )
110 PLD( pld [r1, #92] )
111
112 3: PLD( pld [r1, #124] )
113 4: ldr8w r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
114 subs r2, r2, #32
115 str8w r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
116 bge 3b
117 PLD( cmn r2, #96 )
118 PLD( bge 4b )
119
120 5: ands ip, r2, #28
121 rsb ip, ip, #32
122 #if LDR1W_SHIFT > 0
123 lsl ip, ip, #LDR1W_SHIFT
124 #endif
125 addne pc, pc, ip @ C is always clear here
126 b 7f
127 6:
128 .rept (1 << LDR1W_SHIFT)
129 W(nop)
130 .endr
131 ldr1w r1, r3, abort=20f
132 ldr1w r1, r4, abort=20f
133 ldr1w r1, r5, abort=20f
134 ldr1w r1, r6, abort=20f
135 ldr1w r1, r7, abort=20f
136 ldr1w r1, r8, abort=20f
137 ldr1w r1, lr, abort=20f
138
139 #if LDR1W_SHIFT < STR1W_SHIFT
140 lsl ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
141 #elif LDR1W_SHIFT > STR1W_SHIFT
142 lsr ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
143 #endif
144 add pc, pc, ip
145 nop
146 .rept (1 << STR1W_SHIFT)
147 W(nop)
148 .endr
149 str1w r0, r3, abort=20f
150 str1w r0, r4, abort=20f
151 str1w r0, r5, abort=20f
152 str1w r0, r6, abort=20f
153 str1w r0, r7, abort=20f
154 str1w r0, r8, abort=20f
155 str1w r0, lr, abort=20f
156
157 CALGN( bcs 2b )
158
159 7: ldmfd sp!, {r5 - r8}
160 UNWIND( .fnend ) @ end of second stmfd block
161
162 UNWIND( .fnstart )
163 usave r4, lr @ still in first stmdb block
164 8: movs r2, r2, lsl #31
165 ldr1b r1, r3, ne, abort=21f
166 ldr1b r1, r4, cs, abort=21f
167 ldr1b r1, ip, cs, abort=21f
168 str1b r0, r3, ne, abort=21f
169 str1b r0, r4, cs, abort=21f
170 str1b r0, ip, cs, abort=21f
171
172 exit r4, pc
173
174 9: rsb ip, ip, #4
175 cmp ip, #2
176 ldr1b r1, r3, gt, abort=21f
177 ldr1b r1, r4, ge, abort=21f
178 ldr1b r1, lr, abort=21f
179 str1b r0, r3, gt, abort=21f
180 str1b r0, r4, ge, abort=21f
181 subs r2, r2, ip
182 str1b r0, lr, abort=21f
183 blt 8b
184 ands ip, r1, #3
185 beq 1b
186
187 10: bic r1, r1, #3
188 cmp ip, #2
189 ldr1w r1, lr, abort=21f
190 beq 17f
191 bgt 18f
192 UNWIND( .fnend )
193
194
195 .macro forward_copy_shift pull push
196
197 UNWIND( .fnstart )
198 usave r4, lr @ still in first stmdb block
199 subs r2, r2, #28
200 blt 14f
201
202 CALGN( ands ip, r0, #31 )
203 CALGN( rsb ip, ip, #32 )
204 CALGN( sbcsne r4, ip, r2 ) @ C is always set here
205 CALGN( subcc r2, r2, ip )
206 CALGN( bcc 15f )
207
208 11: stmfd sp!, {r5 - r9}
209 UNWIND( .fnend )
210
211 UNWIND( .fnstart )
212 usave r4, lr
213 UNWIND( .save {r5 - r9} ) @ in new second stmfd block
214 PLD( pld [r1, #0] )
215 PLD( subs r2, r2, #96 )
216 PLD( pld [r1, #28] )
217 PLD( blt 13f )
218 PLD( pld [r1, #60] )
219 PLD( pld [r1, #92] )
220
221 12: PLD( pld [r1, #124] )
222 13: ldr4w r1, r4, r5, r6, r7, abort=19f
223 mov r3, lr, lspull #\pull
224 subs r2, r2, #32
225 ldr4w r1, r8, r9, ip, lr, abort=19f
226 orr r3, r3, r4, lspush #\push
227 mov r4, r4, lspull #\pull
228 orr r4, r4, r5, lspush #\push
229 mov r5, r5, lspull #\pull
230 orr r5, r5, r6, lspush #\push
231 mov r6, r6, lspull #\pull
232 orr r6, r6, r7, lspush #\push
233 mov r7, r7, lspull #\pull
234 orr r7, r7, r8, lspush #\push
235 mov r8, r8, lspull #\pull
236 orr r8, r8, r9, lspush #\push
237 mov r9, r9, lspull #\pull
238 orr r9, r9, ip, lspush #\push
239 mov ip, ip, lspull #\pull
240 orr ip, ip, lr, lspush #\push
241 str8w r0, r3, r4, r5, r6, r7, r8, r9, ip, abort=19f
242 bge 12b
243 PLD( cmn r2, #96 )
244 PLD( bge 13b )
245
246 ldmfd sp!, {r5 - r9}
247 UNWIND( .fnend ) @ end of the second stmfd block
248
249 UNWIND( .fnstart )
250 usave r4, lr @ still in first stmdb block
251 14: ands ip, r2, #28
252 beq 16f
253
254 15: mov r3, lr, lspull #\pull
255 ldr1w r1, lr, abort=21f
256 subs ip, ip, #4
257 orr r3, r3, lr, lspush #\push
258 str1w r0, r3, abort=21f
259 bgt 15b
260 CALGN( cmp r2, #0 )
261 CALGN( bge 11b )
262
263 16: sub r1, r1, #(\push / 8)
264 b 8b
265 UNWIND( .fnend )
266
267 .endm
268
269
270 forward_copy_shift pull=8 push=24
271
272 17: forward_copy_shift pull=16 push=16
273
274 18: forward_copy_shift pull=24 push=8
275
276
277 /*
278 * Abort preamble and completion macros.
279 * If a fixup handler is required then those macros must surround it.
280 * It is assumed that the fixup code will handle the private part of
281 * the exit macro.
282 */
283
284 .macro copy_abort_preamble
285 19: ldmfd sp!, {r5 - r9}
286 b 21f
287 20: ldmfd sp!, {r5 - r8}
288 21:
289 .endm
290
291 .macro copy_abort_end
292 ldmfd sp!, {r4, pc}
293 .endm
294