1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * linux/arch/arm/boot/compressed/head.S
4 *
5 * Copyright (C) 1996-2002 Russell King
6 * Copyright (C) 2004 Hyok S. Choi (MPU support)
7 */
8 #include <linux/linkage.h>
9 #include <asm/assembler.h>
10 #include <asm/v7m.h>
11
12 #include "efi-header.S"
13
14 AR_CLASS( .arch armv7-a )
15 M_CLASS( .arch armv7-m )
16
17 /*
18 * Debugging stuff
19 *
20 * Note that these macros must not contain any code which is not
21 * 100% relocatable. Any attempt to do so will result in a crash.
22 * Please select one of the following when turning on debugging.
23 */
24 #ifdef DEBUG
25
26 #if defined(CONFIG_DEBUG_ICEDCC)
27
28 #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) || defined(CONFIG_CPU_V7)
29 .macro loadsp, rb, tmp1, tmp2
30 .endm
31 .macro writeb, ch, rb
32 mcr p14, 0, \ch, c0, c5, 0
33 .endm
34 #elif defined(CONFIG_CPU_XSCALE)
35 .macro loadsp, rb, tmp1, tmp2
36 .endm
37 .macro writeb, ch, rb
38 mcr p14, 0, \ch, c8, c0, 0
39 .endm
40 #else
41 .macro loadsp, rb, tmp1, tmp2
42 .endm
43 .macro writeb, ch, rb
44 mcr p14, 0, \ch, c1, c0, 0
45 .endm
46 #endif
47
48 #else
49
50 #include CONFIG_DEBUG_LL_INCLUDE
51
52 .macro writeb, ch, rb
53 senduart \ch, \rb
54 .endm
55
56 #if defined(CONFIG_ARCH_SA1100)
57 .macro loadsp, rb, tmp1, tmp2
58 mov \rb, #0x80000000 @ physical base address
59 #ifdef CONFIG_DEBUG_LL_SER3
60 add \rb, \rb, #0x00050000 @ Ser3
61 #else
62 add \rb, \rb, #0x00010000 @ Ser1
63 #endif
64 .endm
65 #else
66 .macro loadsp, rb, tmp1, tmp2
67 addruart \rb, \tmp1, \tmp2
68 .endm
69 #endif
70 #endif
71 #endif
72
73 .macro kputc,val
74 mov r0, \val
75 bl putc
76 .endm
77
78 .macro kphex,val,len
79 mov r0, \val
80 mov r1, #\len
81 bl phex
82 .endm
83
84 .macro debug_reloc_start
85 #ifdef DEBUG
86 kputc #'\n'
87 kphex r6, 8 /* processor id */
88 kputc #':'
89 kphex r7, 8 /* architecture id */
90 #ifdef CONFIG_CPU_CP15
91 kputc #':'
92 mrc p15, 0, r0, c1, c0
93 kphex r0, 8 /* control reg */
94 #endif
95 kputc #'\n'
96 kphex r5, 8 /* decompressed kernel start */
97 kputc #'-'
98 kphex r9, 8 /* decompressed kernel end */
99 kputc #'>'
100 kphex r4, 8 /* kernel execution address */
101 kputc #'\n'
102 #endif
103 .endm
104
105 .macro debug_reloc_end
106 #ifdef DEBUG
107 kphex r5, 8 /* end of kernel */
108 kputc #'\n'
109 mov r0, r4
110 bl memdump /* dump 256 bytes at start of kernel */
111 #endif
112 .endm
113
114 /*
115 * Debug kernel copy by printing the memory addresses involved
116 */
117 .macro dbgkc, begin, end, cbegin, cend
118 #ifdef DEBUG
119 kputc #'\n'
120 kputc #'C'
121 kputc #':'
122 kputc #'0'
123 kputc #'x'
124 kphex \begin, 8 /* Start of compressed kernel */
125 kputc #'-'
126 kputc #'0'
127 kputc #'x'
128 kphex \end, 8 /* End of compressed kernel */
129 kputc #'-'
130 kputc #'>'
131 kputc #'0'
132 kputc #'x'
133 kphex \cbegin, 8 /* Start of kernel copy */
134 kputc #'-'
135 kputc #'0'
136 kputc #'x'
137 kphex \cend, 8 /* End of kernel copy */
138 kputc #'\n'
139 kputc #'\r'
140 #endif
141 .endm
142
143 .section ".start", #alloc, #execinstr
144 /*
145 * sort out different calling conventions
146 */
147 .align
148 /*
149 * Always enter in ARM state for CPUs that support the ARM ISA.
150 * As of today (2014) that's exactly the members of the A and R
151 * classes.
152 */
153 AR_CLASS( .arm )
154 start:
155 .type start,#function
156 /*
157 * These 7 nops along with the 1 nop immediately below for
158 * !THUMB2 form 8 nops that make the compressed kernel bootable
159 * on legacy ARM systems that were assuming the kernel in a.out
160 * binary format. The boot loaders on these systems would
161 * jump 32 bytes into the image to skip the a.out header.
162 * with these 8 nops filling exactly 32 bytes, things still
163 * work as expected on these legacy systems. Thumb2 mode keeps
164 * 7 of the nops as it turns out that some boot loaders
165 * were patching the initial instructions of the kernel, i.e
166 * had started to exploit this "patch area".
167 */
168 .rept 7
169 __nop
170 .endr
171 #ifndef CONFIG_THUMB2_KERNEL
172 __nop
173 #else
174 AR_CLASS( sub pc, pc, #3 ) @ A/R: switch to Thumb2 mode
175 M_CLASS( nop.w ) @ M: already in Thumb2 mode
176 .thumb
177 #endif
178 W(b) 1f
179
180 .word _magic_sig @ Magic numbers to help the loader
181 .word _magic_start @ absolute load/run zImage address
182 .word _magic_end @ zImage end address
183 .word 0x04030201 @ endianness flag
184 .word 0x45454545 @ another magic number to indicate
185 .word _magic_table @ additional data table
186
187 __EFI_HEADER
188 1:
189 ARM_BE8( setend be ) @ go BE8 if compiled for BE8
190 AR_CLASS( mrs r9, cpsr )
191 #ifdef CONFIG_ARM_VIRT_EXT
192 bl __hyp_stub_install @ get into SVC mode, reversibly
193 #endif
194 mov r7, r1 @ save architecture ID
195 mov r8, r2 @ save atags pointer
196
197 #ifndef CONFIG_CPU_V7M
198 /*
199 * Booting from Angel - need to enter SVC mode and disable
200 * FIQs/IRQs (numeric definitions from angel arm.h source).
201 * We only do this if we were in user mode on entry.
202 */
203 mrs r2, cpsr @ get current mode
204 tst r2, #3 @ not user?
205 bne not_angel
206 mov r0, #0x17 @ angel_SWIreason_EnterSVC
207 ARM( swi 0x123456 ) @ angel_SWI_ARM
208 THUMB( svc 0xab ) @ angel_SWI_THUMB
209 not_angel:
210 safe_svcmode_maskall r0
211 msr spsr_cxsf, r9 @ Save the CPU boot mode in
212 @ SPSR
213 #endif
214 /*
215 * Note that some cache flushing and other stuff may
216 * be needed here - is there an Angel SWI call for this?
217 */
218
219 /*
220 * some architecture specific code can be inserted
221 * by the linker here, but it should preserve r7, r8, and r9.
222 */
223
224 .text
225
226 #ifdef CONFIG_AUTO_ZRELADDR
227 /*
228 * Find the start of physical memory. As we are executing
229 * without the MMU on, we are in the physical address space.
230 * We just need to get rid of any offset by aligning the
231 * address.
232 *
233 * This alignment is a balance between the requirements of
234 * different platforms - we have chosen 128MB to allow
235 * platforms which align the start of their physical memory
236 * to 128MB to use this feature, while allowing the zImage
237 * to be placed within the first 128MB of memory on other
238 * platforms. Increasing the alignment means we place
239 * stricter alignment requirements on the start of physical
240 * memory, but relaxing it means that we break people who
241 * are already placing their zImage in (eg) the top 64MB
242 * of this range.
243 */
244 mov r4, pc
245 and r4, r4, #0xf8000000
246 /* Determine final kernel image address. */
247 add r4, r4, #TEXT_OFFSET
248 #else
249 ldr r4, =zreladdr
250 #endif
251
252 /*
253 * Set up a page table only if it won't overwrite ourself.
254 * That means r4 < pc || r4 - 16k page directory > &_end.
255 * Given that r4 > &_end is most unfrequent, we add a rough
256 * additional 1MB of room for a possible appended DTB.
257 */
258 mov r0, pc
259 cmp r0, r4
260 ldrcc r0, LC0+32
261 addcc r0, r0, pc
262 cmpcc r4, r0
263 orrcc r4, r4, #1 @ remember we skipped cache_on
264 blcs cache_on
265
266 restart: adr r0, LC0
267 ldmia r0, {r1, r2, r3, r6, r10, r11, r12}
268 ldr sp, [r0, #28]
269
270 /*
271 * We might be running at a different address. We need
272 * to fix up various pointers.
273 */
274 sub r0, r0, r1 @ calculate the delta offset
275 add r6, r6, r0 @ _edata
276 add r10, r10, r0 @ inflated kernel size location
277
278 /*
279 * The kernel build system appends the size of the
280 * decompressed kernel at the end of the compressed data
281 * in little-endian form.
282 */
283 ldrb r9, [r10, #0]
284 ldrb lr, [r10, #1]
285 orr r9, r9, lr, lsl #8
286 ldrb lr, [r10, #2]
287 ldrb r10, [r10, #3]
288 orr r9, r9, lr, lsl #16
289 orr r9, r9, r10, lsl #24
290
291 #ifndef CONFIG_ZBOOT_ROM
292 /* malloc space is above the relocated stack (64k max) */
293 add sp, sp, r0
294 add r10, sp, #0x10000
295 #else
296 /*
297 * With ZBOOT_ROM the bss/stack is non relocatable,
298 * but someone could still run this code from RAM,
299 * in which case our reference is _edata.
300 */
301 mov r10, r6
302 #endif
303
304 mov r5, #0 @ init dtb size to 0
305 #ifdef CONFIG_ARM_APPENDED_DTB
306 /*
307 * r0 = delta
308 * r2 = BSS start
309 * r3 = BSS end
310 * r4 = final kernel address (possibly with LSB set)
311 * r5 = appended dtb size (still unknown)
312 * r6 = _edata
313 * r7 = architecture ID
314 * r8 = atags/device tree pointer
315 * r9 = size of decompressed image
316 * r10 = end of this image, including bss/stack/malloc space if non XIP
317 * r11 = GOT start
318 * r12 = GOT end
319 * sp = stack pointer
320 *
321 * if there are device trees (dtb) appended to zImage, advance r10 so that the
322 * dtb data will get relocated along with the kernel if necessary.
323 */
324
325 ldr lr, [r6, #0]
326 #ifndef __ARMEB__
327 ldr r1, =0xedfe0dd0 @ sig is 0xd00dfeed big endian
328 #else
329 ldr r1, =0xd00dfeed
330 #endif
331 cmp lr, r1
332 bne dtb_check_done @ not found
333
334 #ifdef CONFIG_ARM_ATAG_DTB_COMPAT
335 /*
336 * OK... Let's do some funky business here.
337 * If we do have a DTB appended to zImage, and we do have
338 * an ATAG list around, we want the later to be translated
339 * and folded into the former here. No GOT fixup has occurred
340 * yet, but none of the code we're about to call uses any
341 * global variable.
342 */
343
344 /* Get the initial DTB size */
345 ldr r5, [r6, #4]
346 #ifndef __ARMEB__
347 /* convert to little endian */
348 eor r1, r5, r5, ror #16
349 bic r1, r1, #0x00ff0000
350 mov r5, r5, ror #8
351 eor r5, r5, r1, lsr #8
352 #endif
353 /* 50% DTB growth should be good enough */
354 add r5, r5, r5, lsr #1
355 /* preserve 64-bit alignment */
356 add r5, r5, #7
357 bic r5, r5, #7
358 /* clamp to 32KB min and 1MB max */
359 cmp r5, #(1 << 15)
360 movlo r5, #(1 << 15)
361 cmp r5, #(1 << 20)
362 movhi r5, #(1 << 20)
363 /* temporarily relocate the stack past the DTB work space */
364 add sp, sp, r5
365
366 stmfd sp!, {r0-r3, ip, lr}
367 mov r0, r8
368 mov r1, r6
369 mov r2, r5
370 bl atags_to_fdt
371
372 /*
373 * If returned value is 1, there is no ATAG at the location
374 * pointed by r8. Try the typical 0x100 offset from start
375 * of RAM and hope for the best.
376 */
377 cmp r0, #1
378 sub r0, r4, #TEXT_OFFSET
379 bic r0, r0, #1
380 add r0, r0, #0x100
381 mov r1, r6
382 mov r2, r5
383 bleq atags_to_fdt
384
385 ldmfd sp!, {r0-r3, ip, lr}
386 sub sp, sp, r5
387 #endif
388
389 mov r8, r6 @ use the appended device tree
390
391 /*
392 * Make sure that the DTB doesn't end up in the final
393 * kernel's .bss area. To do so, we adjust the decompressed
394 * kernel size to compensate if that .bss size is larger
395 * than the relocated code.
396 */
397 ldr r5, =_kernel_bss_size
398 adr r1, wont_overwrite
399 sub r1, r6, r1
400 subs r1, r5, r1
401 addhi r9, r9, r1
402
403 /* Get the current DTB size */
404 ldr r5, [r6, #4]
405 #ifndef __ARMEB__
406 /* convert r5 (dtb size) to little endian */
407 eor r1, r5, r5, ror #16
408 bic r1, r1, #0x00ff0000
409 mov r5, r5, ror #8
410 eor r5, r5, r1, lsr #8
411 #endif
412
413 /* preserve 64-bit alignment */
414 add r5, r5, #7
415 bic r5, r5, #7
416
417 /* relocate some pointers past the appended dtb */
418 add r6, r6, r5
419 add r10, r10, r5
420 add sp, sp, r5
421 dtb_check_done:
422 #endif
423
424 /*
425 * Check to see if we will overwrite ourselves.
426 * r4 = final kernel address (possibly with LSB set)
427 * r9 = size of decompressed image
428 * r10 = end of this image, including bss/stack/malloc space if non XIP
429 * We basically want:
430 * r4 - 16k page directory >= r10 -> OK
431 * r4 + image length <= address of wont_overwrite -> OK
432 * Note: the possible LSB in r4 is harmless here.
433 */
434 add r10, r10, #16384
435 cmp r4, r10
436 bhs wont_overwrite
437 add r10, r4, r9
438 adr r9, wont_overwrite
439 cmp r10, r9
440 bls wont_overwrite
441
442 /*
443 * Relocate ourselves past the end of the decompressed kernel.
444 * r6 = _edata
445 * r10 = end of the decompressed kernel
446 * Because we always copy ahead, we need to do it from the end and go
447 * backward in case the source and destination overlap.
448 */
449 /*
450 * Bump to the next 256-byte boundary with the size of
451 * the relocation code added. This avoids overwriting
452 * ourself when the offset is small.
453 */
454 add r10, r10, #((reloc_code_end - restart + 256) & ~255)
455 bic r10, r10, #255
456
457 /* Get start of code we want to copy and align it down. */
458 adr r5, restart
459 bic r5, r5, #31
460
461 /* Relocate the hyp vector base if necessary */
462 #ifdef CONFIG_ARM_VIRT_EXT
463 mrs r0, spsr
464 and r0, r0, #MODE_MASK
465 cmp r0, #HYP_MODE
466 bne 1f
467
468 /*
469 * Compute the address of the hyp vectors after relocation.
470 * This requires some arithmetic since we cannot directly
471 * reference __hyp_stub_vectors in a PC-relative way.
472 * Call __hyp_set_vectors with the new address so that we
473 * can HVC again after the copy.
474 */
475 0: adr r0, 0b
476 movw r1, #:lower16:__hyp_stub_vectors - 0b
477 movt r1, #:upper16:__hyp_stub_vectors - 0b
478 add r0, r0, r1
479 sub r0, r0, r5
480 add r0, r0, r10
481 bl __hyp_set_vectors
482 1:
483 #endif
484
485 sub r9, r6, r5 @ size to copy
486 add r9, r9, #31 @ rounded up to a multiple
487 bic r9, r9, #31 @ ... of 32 bytes
488 add r6, r9, r5
489 add r9, r9, r10
490
491 #ifdef DEBUG
492 sub r10, r6, r5
493 sub r10, r9, r10
494 /*
495 * We are about to copy the kernel to a new memory area.
496 * The boundaries of the new memory area can be found in
497 * r10 and r9, whilst r5 and r6 contain the boundaries
498 * of the memory we are going to copy.
499 * Calling dbgkc will help with the printing of this
500 * information.
501 */
502 dbgkc r5, r6, r10, r9
503 #endif
504
505 1: ldmdb r6!, {r0 - r3, r10 - r12, lr}
506 cmp r6, r5
507 stmdb r9!, {r0 - r3, r10 - r12, lr}
508 bhi 1b
509
510 /* Preserve offset to relocated code. */
511 sub r6, r9, r6
512
513 #ifndef CONFIG_ZBOOT_ROM
514 /* cache_clean_flush may use the stack, so relocate it */
515 add sp, sp, r6
516 #endif
517
518 bl cache_clean_flush
519
520 badr r0, restart
521 add r0, r0, r6
522 mov pc, r0
523
524 wont_overwrite:
525 /*
526 * If delta is zero, we are running at the address we were linked at.
527 * r0 = delta
528 * r2 = BSS start
529 * r3 = BSS end
530 * r4 = kernel execution address (possibly with LSB set)
531 * r5 = appended dtb size (0 if not present)
532 * r7 = architecture ID
533 * r8 = atags pointer
534 * r11 = GOT start
535 * r12 = GOT end
536 * sp = stack pointer
537 */
538 orrs r1, r0, r5
539 beq not_relocated
540
541 add r11, r11, r0
542 add r12, r12, r0
543
544 #ifndef CONFIG_ZBOOT_ROM
545 /*
546 * If we're running fully PIC === CONFIG_ZBOOT_ROM = n,
547 * we need to fix up pointers into the BSS region.
548 * Note that the stack pointer has already been fixed up.
549 */
550 add r2, r2, r0
551 add r3, r3, r0
552
553 /*
554 * Relocate all entries in the GOT table.
555 * Bump bss entries to _edata + dtb size
556 */
557 1: ldr r1, [r11, #0] @ relocate entries in the GOT
558 add r1, r1, r0 @ This fixes up C references
559 cmp r1, r2 @ if entry >= bss_start &&
560 cmphs r3, r1 @ bss_end > entry
561 addhi r1, r1, r5 @ entry += dtb size
562 str r1, [r11], #4 @ next entry
563 cmp r11, r12
564 blo 1b
565
566 /* bump our bss pointers too */
567 add r2, r2, r5
568 add r3, r3, r5
569
570 #else
571
572 /*
573 * Relocate entries in the GOT table. We only relocate
574 * the entries that are outside the (relocated) BSS region.
575 */
576 1: ldr r1, [r11, #0] @ relocate entries in the GOT
577 cmp r1, r2 @ entry < bss_start ||
578 cmphs r3, r1 @ _end < entry
579 addlo r1, r1, r0 @ table. This fixes up the
580 str r1, [r11], #4 @ C references.
581 cmp r11, r12
582 blo 1b
583 #endif
584
585 not_relocated: mov r0, #0
586 1: str r0, [r2], #4 @ clear bss
587 str r0, [r2], #4
588 str r0, [r2], #4
589 str r0, [r2], #4
590 cmp r2, r3
591 blo 1b
592
593 /*
594 * Did we skip the cache setup earlier?
595 * That is indicated by the LSB in r4.
596 * Do it now if so.
597 */
598 tst r4, #1
599 bic r4, r4, #1
600 blne cache_on
601
602 /*
603 * The C runtime environment should now be setup sufficiently.
604 * Set up some pointers, and start decompressing.
605 * r4 = kernel execution address
606 * r7 = architecture ID
607 * r8 = atags pointer
608 */
609 mov r0, r4
610 mov r1, sp @ malloc space above stack
611 add r2, sp, #0x10000 @ 64k max
612 mov r3, r7
613 bl decompress_kernel
614 bl cache_clean_flush
615 bl cache_off
616
617 #ifdef CONFIG_ARM_VIRT_EXT
618 mrs r0, spsr @ Get saved CPU boot mode
619 and r0, r0, #MODE_MASK
620 cmp r0, #HYP_MODE @ if not booted in HYP mode...
621 bne __enter_kernel @ boot kernel directly
622
623 adr r12, .L__hyp_reentry_vectors_offset
624 ldr r0, [r12]
625 add r0, r0, r12
626
627 bl __hyp_set_vectors
628 __HVC(0) @ otherwise bounce to hyp mode
629
630 b . @ should never be reached
631
632 .align 2
633 .L__hyp_reentry_vectors_offset: .long __hyp_reentry_vectors - .
634 #else
635 b __enter_kernel
636 #endif
637
638 .align 2
639 .type LC0, #object
640 LC0: .word LC0 @ r1
641 .word __bss_start @ r2
642 .word _end @ r3
643 .word _edata @ r6
644 .word input_data_end - 4 @ r10 (inflated size location)
645 .word _got_start @ r11
646 .word _got_end @ ip
647 .word .L_user_stack_end @ sp
648 .word _end - restart + 16384 + 1024*1024
649 .size LC0, . - LC0
650
651 #ifdef CONFIG_ARCH_RPC
652 .globl params
653 params: ldr r0, =0x10000100 @ params_phys for RPC
654 mov pc, lr
655 .ltorg
656 .align
657 #endif
658
659 /*
660 * Turn on the cache. We need to setup some page tables so that we
661 * can have both the I and D caches on.
662 *
663 * We place the page tables 16k down from the kernel execution address,
664 * and we hope that nothing else is using it. If we're using it, we
665 * will go pop!
666 *
667 * On entry,
668 * r4 = kernel execution address
669 * r7 = architecture number
670 * r8 = atags pointer
671 * On exit,
672 * r0, r1, r2, r3, r9, r10, r12 corrupted
673 * This routine must preserve:
674 * r4, r7, r8
675 */
676 .align 5
677 cache_on: mov r3, #8 @ cache_on function
678 b call_cache_fn
679
680 /*
681 * Initialize the highest priority protection region, PR7
682 * to cover all 32bit address and cacheable and bufferable.
683 */
684 __armv4_mpu_cache_on:
685 mov r0, #0x3f @ 4G, the whole
686 mcr p15, 0, r0, c6, c7, 0 @ PR7 Area Setting
687 mcr p15, 0, r0, c6, c7, 1
688
689 mov r0, #0x80 @ PR7
690 mcr p15, 0, r0, c2, c0, 0 @ D-cache on
691 mcr p15, 0, r0, c2, c0, 1 @ I-cache on
692 mcr p15, 0, r0, c3, c0, 0 @ write-buffer on
693
694 mov r0, #0xc000
695 mcr p15, 0, r0, c5, c0, 1 @ I-access permission
696 mcr p15, 0, r0, c5, c0, 0 @ D-access permission
697
698 mov r0, #0
699 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
700 mcr p15, 0, r0, c7, c5, 0 @ flush(inval) I-Cache
701 mcr p15, 0, r0, c7, c6, 0 @ flush(inval) D-Cache
702 mrc p15, 0, r0, c1, c0, 0 @ read control reg
703 @ ...I .... ..D. WC.M
704 orr r0, r0, #0x002d @ .... .... ..1. 11.1
705 orr r0, r0, #0x1000 @ ...1 .... .... ....
706
707 mcr p15, 0, r0, c1, c0, 0 @ write control reg
708
709 mov r0, #0
710 mcr p15, 0, r0, c7, c5, 0 @ flush(inval) I-Cache
711 mcr p15, 0, r0, c7, c6, 0 @ flush(inval) D-Cache
712 mov pc, lr
713
714 __armv3_mpu_cache_on:
715 mov r0, #0x3f @ 4G, the whole
716 mcr p15, 0, r0, c6, c7, 0 @ PR7 Area Setting
717
718 mov r0, #0x80 @ PR7
719 mcr p15, 0, r0, c2, c0, 0 @ cache on
720 mcr p15, 0, r0, c3, c0, 0 @ write-buffer on
721
722 mov r0, #0xc000
723 mcr p15, 0, r0, c5, c0, 0 @ access permission
724
725 mov r0, #0
726 mcr p15, 0, r0, c7, c0, 0 @ invalidate whole cache v3
727 /*
728 * ?? ARMv3 MMU does not allow reading the control register,
729 * does this really work on ARMv3 MPU?
730 */
731 mrc p15, 0, r0, c1, c0, 0 @ read control reg
732 @ .... .... .... WC.M
733 orr r0, r0, #0x000d @ .... .... .... 11.1
734 /* ?? this overwrites the value constructed above? */
735 mov r0, #0
736 mcr p15, 0, r0, c1, c0, 0 @ write control reg
737
738 /* ?? invalidate for the second time? */
739 mcr p15, 0, r0, c7, c0, 0 @ invalidate whole cache v3
740 mov pc, lr
741
742 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
743 #define CB_BITS 0x08
744 #else
745 #define CB_BITS 0x0c
746 #endif
747
748 __setup_mmu: sub r3, r4, #16384 @ Page directory size
749 bic r3, r3, #0xff @ Align the pointer
750 bic r3, r3, #0x3f00
751 /*
752 * Initialise the page tables, turning on the cacheable and bufferable
753 * bits for the RAM area only.
754 */
755 mov r0, r3
756 mov r9, r0, lsr #18
757 mov r9, r9, lsl #18 @ start of RAM
758 add r10, r9, #0x10000000 @ a reasonable RAM size
759 mov r1, #0x12 @ XN|U + section mapping
760 orr r1, r1, #3 << 10 @ AP=11
761 add r2, r3, #16384
762 1: cmp r1, r9 @ if virt > start of RAM
763 cmphs r10, r1 @ && end of RAM > virt
764 bic r1, r1, #0x1c @ clear XN|U + C + B
765 orrlo r1, r1, #0x10 @ Set XN|U for non-RAM
766 orrhs r1, r1, r6 @ set RAM section settings
767 str r1, [r0], #4 @ 1:1 mapping
768 add r1, r1, #1048576
769 teq r0, r2
770 bne 1b
771 /*
772 * If ever we are running from Flash, then we surely want the cache
773 * to be enabled also for our execution instance... We map 2MB of it
774 * so there is no map overlap problem for up to 1 MB compressed kernel.
775 * If the execution is in RAM then we would only be duplicating the above.
776 */
777 orr r1, r6, #0x04 @ ensure B is set for this
778 orr r1, r1, #3 << 10
779 mov r2, pc
780 mov r2, r2, lsr #20
781 orr r1, r1, r2, lsl #20
782 add r0, r3, r2, lsl #2
783 str r1, [r0], #4
784 add r1, r1, #1048576
785 str r1, [r0]
786 mov pc, lr
787 ENDPROC(__setup_mmu)
788
789 @ Enable unaligned access on v6, to allow better code generation
790 @ for the decompressor C code:
791 __armv6_mmu_cache_on:
792 mrc p15, 0, r0, c1, c0, 0 @ read SCTLR
793 bic r0, r0, #2 @ A (no unaligned access fault)
794 orr r0, r0, #1 << 22 @ U (v6 unaligned access model)
795 mcr p15, 0, r0, c1, c0, 0 @ write SCTLR
796 b __armv4_mmu_cache_on
797
798 __arm926ejs_mmu_cache_on:
799 #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH
800 mov r0, #4 @ put dcache in WT mode
801 mcr p15, 7, r0, c15, c0, 0
802 #endif
803
804 __armv4_mmu_cache_on:
805 mov r12, lr
806 #ifdef CONFIG_MMU
807 mov r6, #CB_BITS | 0x12 @ U
808 bl __setup_mmu
809 mov r0, #0
810 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
811 mcr p15, 0, r0, c8, c7, 0 @ flush I,D TLBs
812 mrc p15, 0, r0, c1, c0, 0 @ read control reg
813 orr r0, r0, #0x5000 @ I-cache enable, RR cache replacement
814 orr r0, r0, #0x0030
815 ARM_BE8( orr r0, r0, #1 << 25 ) @ big-endian page tables
816 bl __common_mmu_cache_on
817 mov r0, #0
818 mcr p15, 0, r0, c8, c7, 0 @ flush I,D TLBs
819 #endif
820 mov pc, r12
821
822 __armv7_mmu_cache_on:
823 mov r12, lr
824 #ifdef CONFIG_MMU
825 mrc p15, 0, r11, c0, c1, 4 @ read ID_MMFR0
826 tst r11, #0xf @ VMSA
827 movne r6, #CB_BITS | 0x02 @ !XN
828 blne __setup_mmu
829 mov r0, #0
830 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
831 tst r11, #0xf @ VMSA
832 mcrne p15, 0, r0, c8, c7, 0 @ flush I,D TLBs
833 #endif
834 mrc p15, 0, r0, c1, c0, 0 @ read control reg
835 bic r0, r0, #1 << 28 @ clear SCTLR.TRE
836 orr r0, r0, #0x5000 @ I-cache enable, RR cache replacement
837 orr r0, r0, #0x003c @ write buffer
838 bic r0, r0, #2 @ A (no unaligned access fault)
839 orr r0, r0, #1 << 22 @ U (v6 unaligned access model)
840 @ (needed for ARM1176)
841 #ifdef CONFIG_MMU
842 ARM_BE8( orr r0, r0, #1 << 25 ) @ big-endian page tables
843 mrcne p15, 0, r6, c2, c0, 2 @ read ttb control reg
844 orrne r0, r0, #1 @ MMU enabled
845 movne r1, #0xfffffffd @ domain 0 = client
846 bic r6, r6, #1 << 31 @ 32-bit translation system
847 bic r6, r6, #(7 << 0) | (1 << 4) @ use only ttbr0
848 mcrne p15, 0, r3, c2, c0, 0 @ load page table pointer
849 mcrne p15, 0, r1, c3, c0, 0 @ load domain access control
850 mcrne p15, 0, r6, c2, c0, 2 @ load ttb control
851 #endif
852 mcr p15, 0, r0, c7, c5, 4 @ ISB
853 mcr p15, 0, r0, c1, c0, 0 @ load control register
854 mrc p15, 0, r0, c1, c0, 0 @ and read it back
855 mov r0, #0
856 mcr p15, 0, r0, c7, c5, 4 @ ISB
857 mov pc, r12
858
859 __fa526_cache_on:
860 mov r12, lr
861 mov r6, #CB_BITS | 0x12 @ U
862 bl __setup_mmu
863 mov r0, #0
864 mcr p15, 0, r0, c7, c7, 0 @ Invalidate whole cache
865 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
866 mcr p15, 0, r0, c8, c7, 0 @ flush UTLB
867 mrc p15, 0, r0, c1, c0, 0 @ read control reg
868 orr r0, r0, #0x1000 @ I-cache enable
869 bl __common_mmu_cache_on
870 mov r0, #0
871 mcr p15, 0, r0, c8, c7, 0 @ flush UTLB
872 mov pc, r12
873
874 __common_mmu_cache_on:
875 #ifndef CONFIG_THUMB2_KERNEL
876 #ifndef DEBUG
877 orr r0, r0, #0x000d @ Write buffer, mmu
878 #endif
879 mov r1, #-1
880 mcr p15, 0, r3, c2, c0, 0 @ load page table pointer
881 mcr p15, 0, r1, c3, c0, 0 @ load domain access control
882 b 1f
883 .align 5 @ cache line aligned
884 1: mcr p15, 0, r0, c1, c0, 0 @ load control register
885 mrc p15, 0, r0, c1, c0, 0 @ and read it back to
886 sub pc, lr, r0, lsr #32 @ properly flush pipeline
887 #endif
888
889 #define PROC_ENTRY_SIZE (4*5)
890
891 /*
892 * Here follow the relocatable cache support functions for the
893 * various processors. This is a generic hook for locating an
894 * entry and jumping to an instruction at the specified offset
895 * from the start of the block. Please note this is all position
896 * independent code.
897 *
898 * r1 = corrupted
899 * r2 = corrupted
900 * r3 = block offset
901 * r9 = corrupted
902 * r12 = corrupted
903 */
904
905 call_cache_fn: adr r12, proc_types
906 #ifdef CONFIG_CPU_CP15
907 mrc p15, 0, r9, c0, c0 @ get processor ID
908 #elif defined(CONFIG_CPU_V7M)
909 /*
910 * On v7-M the processor id is located in the V7M_SCB_CPUID
911 * register, but as cache handling is IMPLEMENTATION DEFINED on
912 * v7-M (if existant at all) we just return early here.
913 * If V7M_SCB_CPUID were used the cpu ID functions (i.e.
914 * __armv7_mmu_cache_{on,off,flush}) would be selected which
915 * use cp15 registers that are not implemented on v7-M.
916 */
917 bx lr
918 #else
919 ldr r9, =CONFIG_PROCESSOR_ID
920 #endif
921 1: ldr r1, [r12, #0] @ get value
922 ldr r2, [r12, #4] @ get mask
923 eor r1, r1, r9 @ (real ^ match)
924 tst r1, r2 @ & mask
925 ARM( addeq pc, r12, r3 ) @ call cache function
926 THUMB( addeq r12, r3 )
927 THUMB( moveq pc, r12 ) @ call cache function
928 add r12, r12, #PROC_ENTRY_SIZE
929 b 1b
930
931 /*
932 * Table for cache operations. This is basically:
933 * - CPU ID match
934 * - CPU ID mask
935 * - 'cache on' method instruction
936 * - 'cache off' method instruction
937 * - 'cache flush' method instruction
938 *
939 * We match an entry using: ((real_id ^ match) & mask) == 0
940 *
941 * Writethrough caches generally only need 'on' and 'off'
942 * methods. Writeback caches _must_ have the flush method
943 * defined.
944 */
945 .align 2
946 .type proc_types,#object
947 proc_types:
948 .word 0x41000000 @ old ARM ID
949 .word 0xff00f000
950 mov pc, lr
951 THUMB( nop )
952 mov pc, lr
953 THUMB( nop )
954 mov pc, lr
955 THUMB( nop )
956
957 .word 0x41007000 @ ARM7/710
958 .word 0xfff8fe00
959 mov pc, lr
960 THUMB( nop )
961 mov pc, lr
962 THUMB( nop )
963 mov pc, lr
964 THUMB( nop )
965
966 .word 0x41807200 @ ARM720T (writethrough)
967 .word 0xffffff00
968 W(b) __armv4_mmu_cache_on
969 W(b) __armv4_mmu_cache_off
970 mov pc, lr
971 THUMB( nop )
972
973 .word 0x41007400 @ ARM74x
974 .word 0xff00ff00
975 W(b) __armv3_mpu_cache_on
976 W(b) __armv3_mpu_cache_off
977 W(b) __armv3_mpu_cache_flush
978
979 .word 0x41009400 @ ARM94x
980 .word 0xff00ff00
981 W(b) __armv4_mpu_cache_on
982 W(b) __armv4_mpu_cache_off
983 W(b) __armv4_mpu_cache_flush
984
985 .word 0x41069260 @ ARM926EJ-S (v5TEJ)
986 .word 0xff0ffff0
987 W(b) __arm926ejs_mmu_cache_on
988 W(b) __armv4_mmu_cache_off
989 W(b) __armv5tej_mmu_cache_flush
990
991 .word 0x00007000 @ ARM7 IDs
992 .word 0x0000f000
993 mov pc, lr
994 THUMB( nop )
995 mov pc, lr
996 THUMB( nop )
997 mov pc, lr
998 THUMB( nop )
999
1000 @ Everything from here on will be the new ID system.
1001
1002 .word 0x4401a100 @ sa110 / sa1100
1003 .word 0xffffffe0
1004 W(b) __armv4_mmu_cache_on
1005 W(b) __armv4_mmu_cache_off
1006 W(b) __armv4_mmu_cache_flush
1007
1008 .word 0x6901b110 @ sa1110
1009 .word 0xfffffff0
1010 W(b) __armv4_mmu_cache_on
1011 W(b) __armv4_mmu_cache_off
1012 W(b) __armv4_mmu_cache_flush
1013
1014 .word 0x56056900
1015 .word 0xffffff00 @ PXA9xx
1016 W(b) __armv4_mmu_cache_on
1017 W(b) __armv4_mmu_cache_off
1018 W(b) __armv4_mmu_cache_flush
1019
1020 .word 0x56158000 @ PXA168
1021 .word 0xfffff000
1022 W(b) __armv4_mmu_cache_on
1023 W(b) __armv4_mmu_cache_off
1024 W(b) __armv5tej_mmu_cache_flush
1025
1026 .word 0x56050000 @ Feroceon
1027 .word 0xff0f0000
1028 W(b) __armv4_mmu_cache_on
1029 W(b) __armv4_mmu_cache_off
1030 W(b) __armv5tej_mmu_cache_flush
1031
1032 #ifdef CONFIG_CPU_FEROCEON_OLD_ID
1033 /* this conflicts with the standard ARMv5TE entry */
1034 .long 0x41009260 @ Old Feroceon
1035 .long 0xff00fff0
1036 b __armv4_mmu_cache_on
1037 b __armv4_mmu_cache_off
1038 b __armv5tej_mmu_cache_flush
1039 #endif
1040
1041 .word 0x66015261 @ FA526
1042 .word 0xff01fff1
1043 W(b) __fa526_cache_on
1044 W(b) __armv4_mmu_cache_off
1045 W(b) __fa526_cache_flush
1046
1047 @ These match on the architecture ID
1048
1049 .word 0x00020000 @ ARMv4T
1050 .word 0x000f0000
1051 W(b) __armv4_mmu_cache_on
1052 W(b) __armv4_mmu_cache_off
1053 W(b) __armv4_mmu_cache_flush
1054
1055 .word 0x00050000 @ ARMv5TE
1056 .word 0x000f0000
1057 W(b) __armv4_mmu_cache_on
1058 W(b) __armv4_mmu_cache_off
1059 W(b) __armv4_mmu_cache_flush
1060
1061 .word 0x00060000 @ ARMv5TEJ
1062 .word 0x000f0000
1063 W(b) __armv4_mmu_cache_on
1064 W(b) __armv4_mmu_cache_off
1065 W(b) __armv5tej_mmu_cache_flush
1066
1067 .word 0x0007b000 @ ARMv6
1068 .word 0x000ff000
1069 W(b) __armv6_mmu_cache_on
1070 W(b) __armv4_mmu_cache_off
1071 W(b) __armv6_mmu_cache_flush
1072
1073 .word 0x000f0000 @ new CPU Id
1074 .word 0x000f0000
1075 W(b) __armv7_mmu_cache_on
1076 W(b) __armv7_mmu_cache_off
1077 W(b) __armv7_mmu_cache_flush
1078
1079 .word 0 @ unrecognised type
1080 .word 0
1081 mov pc, lr
1082 THUMB( nop )
1083 mov pc, lr
1084 THUMB( nop )
1085 mov pc, lr
1086 THUMB( nop )
1087
1088 .size proc_types, . - proc_types
1089
1090 /*
1091 * If you get a "non-constant expression in ".if" statement"
1092 * error from the assembler on this line, check that you have
1093 * not accidentally written a "b" instruction where you should
1094 * have written W(b).
1095 */
1096 .if (. - proc_types) % PROC_ENTRY_SIZE != 0
1097 .error "The size of one or more proc_types entries is wrong."
1098 .endif
1099
1100 /*
1101 * Turn off the Cache and MMU. ARMv3 does not support
1102 * reading the control register, but ARMv4 does.
1103 *
1104 * On exit,
1105 * r0, r1, r2, r3, r9, r12 corrupted
1106 * This routine must preserve:
1107 * r4, r7, r8
1108 */
1109 .align 5
1110 cache_off: mov r3, #12 @ cache_off function
1111 b call_cache_fn
1112
1113 __armv4_mpu_cache_off:
1114 mrc p15, 0, r0, c1, c0
1115 bic r0, r0, #0x000d
1116 mcr p15, 0, r0, c1, c0 @ turn MPU and cache off
1117 mov r0, #0
1118 mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
1119 mcr p15, 0, r0, c7, c6, 0 @ flush D-Cache
1120 mcr p15, 0, r0, c7, c5, 0 @ flush I-Cache
1121 mov pc, lr
1122
1123 __armv3_mpu_cache_off:
1124 mrc p15, 0, r0, c1, c0
1125 bic r0, r0, #0x000d
1126 mcr p15, 0, r0, c1, c0, 0 @ turn MPU and cache off
1127 mov r0, #0
1128 mcr p15, 0, r0, c7, c0, 0 @ invalidate whole cache v3
1129 mov pc, lr
1130
1131 __armv4_mmu_cache_off:
1132 #ifdef CONFIG_MMU
1133 mrc p15, 0, r0, c1, c0
1134 bic r0, r0, #0x000d
1135 mcr p15, 0, r0, c1, c0 @ turn MMU and cache off
1136 mov r0, #0
1137 mcr p15, 0, r0, c7, c7 @ invalidate whole cache v4
1138 mcr p15, 0, r0, c8, c7 @ invalidate whole TLB v4
1139 #endif
1140 mov pc, lr
1141
1142 __armv7_mmu_cache_off:
1143 mrc p15, 0, r0, c1, c0
1144 #ifdef CONFIG_MMU
1145 bic r0, r0, #0x000d
1146 #else
1147 bic r0, r0, #0x000c
1148 #endif
1149 mcr p15, 0, r0, c1, c0 @ turn MMU and cache off
1150 mov r12, lr
1151 bl __armv7_mmu_cache_flush
1152 mov r0, #0
1153 #ifdef CONFIG_MMU
1154 mcr p15, 0, r0, c8, c7, 0 @ invalidate whole TLB
1155 #endif
1156 mcr p15, 0, r0, c7, c5, 6 @ invalidate BTC
1157 mcr p15, 0, r0, c7, c10, 4 @ DSB
1158 mcr p15, 0, r0, c7, c5, 4 @ ISB
1159 mov pc, r12
1160
1161 /*
1162 * Clean and flush the cache to maintain consistency.
1163 *
1164 * On exit,
1165 * r1, r2, r3, r9, r10, r11, r12 corrupted
1166 * This routine must preserve:
1167 * r4, r6, r7, r8
1168 */
1169 .align 5
1170 cache_clean_flush:
1171 mov r3, #16
1172 b call_cache_fn
1173
1174 __armv4_mpu_cache_flush:
1175 tst r4, #1
1176 movne pc, lr
1177 mov r2, #1
1178 mov r3, #0
1179 mcr p15, 0, ip, c7, c6, 0 @ invalidate D cache
1180 mov r1, #7 << 5 @ 8 segments
1181 1: orr r3, r1, #63 << 26 @ 64 entries
1182 2: mcr p15, 0, r3, c7, c14, 2 @ clean & invalidate D index
1183 subs r3, r3, #1 << 26
1184 bcs 2b @ entries 63 to 0
1185 subs r1, r1, #1 << 5
1186 bcs 1b @ segments 7 to 0
1187
1188 teq r2, #0
1189 mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache
1190 mcr p15, 0, ip, c7, c10, 4 @ drain WB
1191 mov pc, lr
1192
1193 __fa526_cache_flush:
1194 tst r4, #1
1195 movne pc, lr
1196 mov r1, #0
1197 mcr p15, 0, r1, c7, c14, 0 @ clean and invalidate D cache
1198 mcr p15, 0, r1, c7, c5, 0 @ flush I cache
1199 mcr p15, 0, r1, c7, c10, 4 @ drain WB
1200 mov pc, lr
1201
1202 __armv6_mmu_cache_flush:
1203 mov r1, #0
1204 tst r4, #1
1205 mcreq p15, 0, r1, c7, c14, 0 @ clean+invalidate D
1206 mcr p15, 0, r1, c7, c5, 0 @ invalidate I+BTB
1207 mcreq p15, 0, r1, c7, c15, 0 @ clean+invalidate unified
1208 mcr p15, 0, r1, c7, c10, 4 @ drain WB
1209 mov pc, lr
1210
1211 __armv7_mmu_cache_flush:
1212 tst r4, #1
1213 bne iflush
1214 mrc p15, 0, r10, c0, c1, 5 @ read ID_MMFR1
1215 tst r10, #0xf << 16 @ hierarchical cache (ARMv7)
1216 mov r10, #0
1217 beq hierarchical
1218 mcr p15, 0, r10, c7, c14, 0 @ clean+invalidate D
1219 b iflush
1220 hierarchical:
1221 mcr p15, 0, r10, c7, c10, 5 @ DMB
1222 stmfd sp!, {r0-r7, r9-r11}
1223 mrc p15, 1, r0, c0, c0, 1 @ read clidr
1224 ands r3, r0, #0x7000000 @ extract loc from clidr
1225 mov r3, r3, lsr #23 @ left align loc bit field
1226 beq finished @ if loc is 0, then no need to clean
1227 mov r10, #0 @ start clean at cache level 0
1228 loop1:
1229 add r2, r10, r10, lsr #1 @ work out 3x current cache level
1230 mov r1, r0, lsr r2 @ extract cache type bits from clidr
1231 and r1, r1, #7 @ mask of the bits for current cache only
1232 cmp r1, #2 @ see what cache we have at this level
1233 blt skip @ skip if no cache, or just i-cache
1234 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
1235 mcr p15, 0, r10, c7, c5, 4 @ isb to sych the new cssr&csidr
1236 mrc p15, 1, r1, c0, c0, 0 @ read the new csidr
1237 and r2, r1, #7 @ extract the length of the cache lines
1238 add r2, r2, #4 @ add 4 (line length offset)
1239 ldr r4, =0x3ff
1240 ands r4, r4, r1, lsr #3 @ find maximum number on the way size
1241 clz r5, r4 @ find bit position of way size increment
1242 ldr r7, =0x7fff
1243 ands r7, r7, r1, lsr #13 @ extract max number of the index size
1244 loop2:
1245 mov r9, r4 @ create working copy of max way size
1246 loop3:
1247 ARM( orr r11, r10, r9, lsl r5 ) @ factor way and cache number into r11
1248 ARM( orr r11, r11, r7, lsl r2 ) @ factor index number into r11
1249 THUMB( lsl r6, r9, r5 )
1250 THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11
1251 THUMB( lsl r6, r7, r2 )
1252 THUMB( orr r11, r11, r6 ) @ factor index number into r11
1253 mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way
1254 subs r9, r9, #1 @ decrement the way
1255 bge loop3
1256 subs r7, r7, #1 @ decrement the index
1257 bge loop2
1258 skip:
1259 add r10, r10, #2 @ increment cache number
1260 cmp r3, r10
1261 bgt loop1
1262 finished:
1263 ldmfd sp!, {r0-r7, r9-r11}
1264 mov r10, #0 @ switch back to cache level 0
1265 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
1266 iflush:
1267 mcr p15, 0, r10, c7, c10, 4 @ DSB
1268 mcr p15, 0, r10, c7, c5, 0 @ invalidate I+BTB
1269 mcr p15, 0, r10, c7, c10, 4 @ DSB
1270 mcr p15, 0, r10, c7, c5, 4 @ ISB
1271 mov pc, lr
1272
1273 __armv5tej_mmu_cache_flush:
1274 tst r4, #1
1275 movne pc, lr
1276 1: mrc p15, 0, r15, c7, c14, 3 @ test,clean,invalidate D cache
1277 bne 1b
1278 mcr p15, 0, r0, c7, c5, 0 @ flush I cache
1279 mcr p15, 0, r0, c7, c10, 4 @ drain WB
1280 mov pc, lr
1281
1282 __armv4_mmu_cache_flush:
1283 tst r4, #1
1284 movne pc, lr
1285 mov r2, #64*1024 @ default: 32K dcache size (*2)
1286 mov r11, #32 @ default: 32 byte line size
1287 mrc p15, 0, r3, c0, c0, 1 @ read cache type
1288 teq r3, r9 @ cache ID register present?
1289 beq no_cache_id
1290 mov r1, r3, lsr #18
1291 and r1, r1, #7
1292 mov r2, #1024
1293 mov r2, r2, lsl r1 @ base dcache size *2
1294 tst r3, #1 << 14 @ test M bit
1295 addne r2, r2, r2, lsr #1 @ +1/2 size if M == 1
1296 mov r3, r3, lsr #12
1297 and r3, r3, #3
1298 mov r11, #8
1299 mov r11, r11, lsl r3 @ cache line size in bytes
1300 no_cache_id:
1301 mov r1, pc
1302 bic r1, r1, #63 @ align to longest cache line
1303 add r2, r1, r2
1304 1:
1305 ARM( ldr r3, [r1], r11 ) @ s/w flush D cache
1306 THUMB( ldr r3, [r1] ) @ s/w flush D cache
1307 THUMB( add r1, r1, r11 )
1308 teq r1, r2
1309 bne 1b
1310
1311 mcr p15, 0, r1, c7, c5, 0 @ flush I cache
1312 mcr p15, 0, r1, c7, c6, 0 @ flush D cache
1313 mcr p15, 0, r1, c7, c10, 4 @ drain WB
1314 mov pc, lr
1315
1316 __armv3_mmu_cache_flush:
1317 __armv3_mpu_cache_flush:
1318 tst r4, #1
1319 movne pc, lr
1320 mov r1, #0
1321 mcr p15, 0, r1, c7, c0, 0 @ invalidate whole cache v3
1322 mov pc, lr
1323
1324 /*
1325 * Various debugging routines for printing hex characters and
1326 * memory, which again must be relocatable.
1327 */
1328 #ifdef DEBUG
1329 .align 2
1330 .type phexbuf,#object
1331 phexbuf: .space 12
1332 .size phexbuf, . - phexbuf
1333
1334 @ phex corrupts {r0, r1, r2, r3}
1335 phex: adr r3, phexbuf
1336 mov r2, #0
1337 strb r2, [r3, r1]
1338 1: subs r1, r1, #1
1339 movmi r0, r3
1340 bmi puts
1341 and r2, r0, #15
1342 mov r0, r0, lsr #4
1343 cmp r2, #10
1344 addge r2, r2, #7
1345 add r2, r2, #'0'
1346 strb r2, [r3, r1]
1347 b 1b
1348
1349 @ puts corrupts {r0, r1, r2, r3}
1350 puts: loadsp r3, r2, r1
1351 1: ldrb r2, [r0], #1
1352 teq r2, #0
1353 moveq pc, lr
1354 2: writeb r2, r3
1355 mov r1, #0x00020000
1356 3: subs r1, r1, #1
1357 bne 3b
1358 teq r2, #'\n'
1359 moveq r2, #'\r'
1360 beq 2b
1361 teq r0, #0
1362 bne 1b
1363 mov pc, lr
1364 @ putc corrupts {r0, r1, r2, r3}
1365 putc:
1366 mov r2, r0
1367 loadsp r3, r1, r0
1368 mov r0, #0
1369 b 2b
1370
1371 @ memdump corrupts {r0, r1, r2, r3, r10, r11, r12, lr}
1372 memdump: mov r12, r0
1373 mov r10, lr
1374 mov r11, #0
1375 2: mov r0, r11, lsl #2
1376 add r0, r0, r12
1377 mov r1, #8
1378 bl phex
1379 mov r0, #':'
1380 bl putc
1381 1: mov r0, #' '
1382 bl putc
1383 ldr r0, [r12, r11, lsl #2]
1384 mov r1, #8
1385 bl phex
1386 and r0, r11, #7
1387 teq r0, #3
1388 moveq r0, #' '
1389 bleq putc
1390 and r0, r11, #7
1391 add r11, r11, #1
1392 teq r0, #7
1393 bne 1b
1394 mov r0, #'\n'
1395 bl putc
1396 cmp r11, #64
1397 blt 2b
1398 mov pc, r10
1399 #endif
1400
1401 .ltorg
1402
1403 #ifdef CONFIG_ARM_VIRT_EXT
1404 .align 5
1405 __hyp_reentry_vectors:
1406 W(b) . @ reset
1407 W(b) . @ undef
1408 W(b) . @ svc
1409 W(b) . @ pabort
1410 W(b) . @ dabort
1411 W(b) __enter_kernel @ hyp
1412 W(b) . @ irq
1413 W(b) . @ fiq
1414 #endif /* CONFIG_ARM_VIRT_EXT */
1415
1416 __enter_kernel:
1417 mov r0, #0 @ must be 0
1418 mov r1, r7 @ restore architecture number
1419 mov r2, r8 @ restore atags pointer
1420 ARM( mov pc, r4 ) @ call kernel
1421 M_CLASS( add r4, r4, #1 ) @ enter in Thumb mode for M class
1422 THUMB( bx r4 ) @ entry point is always ARM for A/R classes
1423
1424 reloc_code_end:
1425
1426 #ifdef CONFIG_EFI_STUB
1427 .align 2
1428 _start: .long start - .
1429
1430 ENTRY(efi_stub_entry)
1431 @ allocate space on stack for passing current zImage address
1432 @ and for the EFI stub to return of new entry point of
1433 @ zImage, as EFI stub may copy the kernel. Pointer address
1434 @ is passed in r2. r0 and r1 are passed through from the
1435 @ EFI firmware to efi_entry
1436 adr ip, _start
1437 ldr r3, [ip]
1438 add r3, r3, ip
1439 stmfd sp!, {r3, lr}
1440 mov r2, sp @ pass zImage address in r2
1441 bl efi_entry
1442
1443 @ Check for error return from EFI stub. r0 has FDT address
1444 @ or error code.
1445 cmn r0, #1
1446 beq efi_load_fail
1447
1448 @ Preserve return value of efi_entry() in r4
1449 mov r4, r0
1450
1451 @ our cache maintenance code relies on CP15 barrier instructions
1452 @ but since we arrived here with the MMU and caches configured
1453 @ by UEFI, we must check that the CP15BEN bit is set in SCTLR.
1454 @ Note that this bit is RAO/WI on v6 and earlier, so the ISB in
1455 @ the enable path will be executed on v7+ only.
1456 mrc p15, 0, r1, c1, c0, 0 @ read SCTLR
1457 tst r1, #(1 << 5) @ CP15BEN bit set?
1458 bne 0f
1459 orr r1, r1, #(1 << 5) @ CP15 barrier instructions
1460 mcr p15, 0, r1, c1, c0, 0 @ write SCTLR
1461 ARM( .inst 0xf57ff06f @ v7+ isb )
1462 THUMB( isb )
1463
1464 0: bl cache_clean_flush
1465 bl cache_off
1466
1467 @ Set parameters for booting zImage according to boot protocol
1468 @ put FDT address in r2, it was returned by efi_entry()
1469 @ r1 is the machine type, and r0 needs to be 0
1470 mov r0, #0
1471 mov r1, #0xFFFFFFFF
1472 mov r2, r4
1473
1474 @ Branch to (possibly) relocated zImage that is in [sp]
1475 ldr lr, [sp]
1476 ldr ip, =start_offset
1477 add lr, lr, ip
1478 mov pc, lr @ no mode switch
1479
1480 efi_load_fail:
1481 @ Return EFI_LOAD_ERROR to EFI firmware on error.
1482 ldr r0, =0x80000001
1483 ldmfd sp!, {ip, pc}
1484 ENDPROC(efi_stub_entry)
1485 #endif
1486
1487 .align
1488 .section ".stack", "aw", %nobits
1489 .L_user_stack: .space 4096
1490 .L_user_stack_end: