1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 #include <linux/errno.h>
18 #include <linux/err.h>
19 #include <linux/sys.h>
20 #include <linux/threads.h>
21 #include <asm/reg.h>
22 #include <asm/page.h>
23 #include <asm/mmu.h>
24 #include <asm/cputable.h>
25 #include <asm/thread_info.h>
26 #include <asm/ppc_asm.h>
27 #include <asm/asm-offsets.h>
28 #include <asm/unistd.h>
29 #include <asm/ptrace.h>
30 #include <asm/export.h>
31 #include <asm/asm-405.h>
32 #include <asm/feature-fixups.h>
33 #include <asm/barrier.h>
34 #include <asm/kup.h>
35 #include <asm/bug.h>
36
37 #include "head_32.h"
38
39
40
41
42
43
44 .align 12
45
46 #ifdef CONFIG_BOOKE
47 .globl mcheck_transfer_to_handler
48 mcheck_transfer_to_handler:
49 mfspr r0,SPRN_DSRR0
50 stw r0,_DSRR0(r11)
51 mfspr r0,SPRN_DSRR1
52 stw r0,_DSRR1(r11)
53
54
55 .globl debug_transfer_to_handler
56 debug_transfer_to_handler:
57 mfspr r0,SPRN_CSRR0
58 stw r0,_CSRR0(r11)
59 mfspr r0,SPRN_CSRR1
60 stw r0,_CSRR1(r11)
61
62
63 .globl crit_transfer_to_handler
64 crit_transfer_to_handler:
65 #ifdef CONFIG_PPC_BOOK3E_MMU
66 mfspr r0,SPRN_MAS0
67 stw r0,MAS0(r11)
68 mfspr r0,SPRN_MAS1
69 stw r0,MAS1(r11)
70 mfspr r0,SPRN_MAS2
71 stw r0,MAS2(r11)
72 mfspr r0,SPRN_MAS3
73 stw r0,MAS3(r11)
74 mfspr r0,SPRN_MAS6
75 stw r0,MAS6(r11)
76 #ifdef CONFIG_PHYS_64BIT
77 mfspr r0,SPRN_MAS7
78 stw r0,MAS7(r11)
79 #endif
80 #endif
81 #ifdef CONFIG_44x
82 mfspr r0,SPRN_MMUCR
83 stw r0,MMUCR(r11)
84 #endif
85 mfspr r0,SPRN_SRR0
86 stw r0,_SRR0(r11)
87 mfspr r0,SPRN_SRR1
88 stw r0,_SRR1(r11)
89
90
91 mfspr r8,SPRN_SPRG_THREAD
92 lwz r0,KSP_LIMIT(r8)
93 stw r0,SAVED_KSP_LIMIT(r11)
94 rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
95 stw r0,KSP_LIMIT(r8)
96
97 #endif
98
99 #ifdef CONFIG_40x
100 .globl crit_transfer_to_handler
101 crit_transfer_to_handler:
102 lwz r0,crit_r10@l(0)
103 stw r0,GPR10(r11)
104 lwz r0,crit_r11@l(0)
105 stw r0,GPR11(r11)
106 mfspr r0,SPRN_SRR0
107 stw r0,crit_srr0@l(0)
108 mfspr r0,SPRN_SRR1
109 stw r0,crit_srr1@l(0)
110
111
112 mfspr r8,SPRN_SPRG_THREAD
113 lwz r0,KSP_LIMIT(r8)
114 stw r0,saved_ksp_limit@l(0)
115 rlwinm r0,r1,0,0,(31 - THREAD_SHIFT)
116 stw r0,KSP_LIMIT(r8)
117
118 #endif
119
120
121
122
123
124
125
126
127 .globl transfer_to_handler_full
128 transfer_to_handler_full:
129 SAVE_NVGPRS(r11)
130
131
132 .globl transfer_to_handler
133 transfer_to_handler:
134 stw r2,GPR2(r11)
135 stw r12,_NIP(r11)
136 stw r9,_MSR(r11)
137 andi. r2,r9,MSR_PR
138 mfctr r12
139 mfspr r2,SPRN_XER
140 stw r12,_CTR(r11)
141 stw r2,_XER(r11)
142 mfspr r12,SPRN_SPRG_THREAD
143 beq 2f
144 addi r2, r12, -THREAD
145 addi r11,r1,STACK_FRAME_OVERHEAD
146 stw r11,PT_REGS(r12)
147 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
148
149
150 lwz r12,THREAD_DBCR0(r12)
151 andis. r12,r12,DBCR0_IDM@h
152 #endif
153 ACCOUNT_CPU_USER_ENTRY(r2, r11, r12)
154 #ifdef CONFIG_PPC_BOOK3S_32
155 kuep_lock r11, r12
156 #endif
157 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
158 beq+ 3f
159
160 li r12,-1
161 mtspr SPRN_DBSR,r12
162 lis r11,global_dbcr0@ha
163 tophys(r11,r11)
164 addi r11,r11,global_dbcr0@l
165 #ifdef CONFIG_SMP
166 lwz r9,TASK_CPU(r2)
167 slwi r9,r9,3
168 add r11,r11,r9
169 #endif
170 lwz r12,0(r11)
171 mtspr SPRN_DBCR0,r12
172 lwz r12,4(r11)
173 addi r12,r12,-1
174 stw r12,4(r11)
175 #endif
176
177 b 3f
178
179 2:
180
181
182 kuap_save_and_lock r11, r12, r9, r2, r6
183 addi r2, r12, -THREAD
184 lwz r9,KSP_LIMIT(r12)
185 cmplw r1,r9
186 ble- stack_ovf
187 5:
188 #if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
189 lwz r12,TI_LOCAL_FLAGS(r2)
190 mtcrf 0x01,r12
191 bt- 31-TLF_NAPPING,4f
192 bt- 31-TLF_SLEEPING,7f
193 #endif
194 .globl transfer_to_handler_cont
195 transfer_to_handler_cont:
196 3:
197 mflr r9
198 tovirt(r2, r2)
199 lwz r11,0(r9)
200 lwz r9,4(r9)
201 #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
202 mtspr SPRN_NRI, r0
203 #endif
204 #ifdef CONFIG_TRACE_IRQFLAGS
205
206
207
208
209
210
211
212
213 tophys(r12, r1)
214 lwz r12,_MSR(r12)
215 andi. r12,r12,MSR_EE
216 bne 1f
217
218
219 #endif
220 mtspr SPRN_SRR0,r11
221 mtspr SPRN_SRR1,r10
222 mtlr r9
223 SYNC
224 RFI
225
226 #ifdef CONFIG_TRACE_IRQFLAGS
227 1:
228
229
230
231 lis r12,reenable_mmu@h
232 ori r12,r12,reenable_mmu@l
233 LOAD_REG_IMMEDIATE(r0, MSR_KERNEL)
234 mtspr SPRN_SRR0,r12
235 mtspr SPRN_SRR1,r0
236 SYNC
237 RFI
238
239 reenable_mmu:
240
241
242
243
244
245
246
247
248
249
250 stwu r1,-32(r1)
251 stw r9,8(r1)
252 stw r11,12(r1)
253 stw r3,16(r1)
254 stw r4,20(r1)
255 stw r5,24(r1)
256
257
258
259
260 1: bl trace_hardirqs_off
261 2: lwz r5,24(r1)
262 lwz r4,20(r1)
263 lwz r3,16(r1)
264 lwz r11,12(r1)
265 lwz r9,8(r1)
266 addi r1,r1,32
267 lwz r0,GPR0(r1)
268 lwz r6,GPR6(r1)
269 lwz r7,GPR7(r1)
270 lwz r8,GPR8(r1)
271 mtctr r11
272 mtlr r9
273 bctr
274 #endif
275
276 #if defined (CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500)
277 4: rlwinm r12,r12,0,~_TLF_NAPPING
278 stw r12,TI_LOCAL_FLAGS(r2)
279 b power_save_ppc32_restore
280
281 7: rlwinm r12,r12,0,~_TLF_SLEEPING
282 stw r12,TI_LOCAL_FLAGS(r2)
283 lwz r9,_MSR(r11)
284 rlwinm r9,r9,0,~MSR_EE
285 lwz r12,_LINK(r11)
286 kuap_restore r11, r2, r3, r4, r5
287 lwz r2, GPR2(r11)
288 b fast_exception_return
289 #endif
290
291
292
293
294
295 stack_ovf:
296
297 lis r12,_end@h
298 ori r12,r12,_end@l
299 cmplw r1,r12
300 ble 5b
301 SAVE_NVGPRS(r11)
302 addi r3,r1,STACK_FRAME_OVERHEAD
303 lis r1,init_thread_union@ha
304 addi r1,r1,init_thread_union@l
305 addi r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
306 lis r9,StackOverflow@ha
307 addi r9,r9,StackOverflow@l
308 LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
309 #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
310 mtspr SPRN_NRI, r0
311 #endif
312 mtspr SPRN_SRR0,r9
313 mtspr SPRN_SRR1,r10
314 SYNC
315 RFI
316
317 #ifdef CONFIG_TRACE_IRQFLAGS
318 trace_syscall_entry_irq_off:
319
320
321
322
323 0: trap
324 EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING
325 bl trace_hardirqs_on
326
327
328 LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE)
329 mtmsr r10
330
331 REST_GPR(0, r1)
332 REST_4GPRS(3, r1)
333 REST_2GPRS(7, r1)
334 b DoSyscall
335 #endif
336
337 .globl transfer_to_syscall
338 transfer_to_syscall:
339 #ifdef CONFIG_TRACE_IRQFLAGS
340 andi. r12,r9,MSR_EE
341 beq- trace_syscall_entry_irq_off
342 #endif
343
344
345
346
347 .stabs "arch/powerpc/kernel/",N_SO,0,0,0f
348 .stabs "entry_32.S",N_SO,0,0,0f
349 0:
350
351 _GLOBAL(DoSyscall)
352 stw r3,ORIG_GPR3(r1)
353 li r12,0
354 stw r12,RESULT(r1)
355 #ifdef CONFIG_TRACE_IRQFLAGS
356
357 mfmsr r11
358 andi. r12,r11,MSR_EE
359
360
361 0: tweqi r12, 0
362 EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING
363 #endif
364 lwz r11,TI_FLAGS(r2)
365 andi. r11,r11,_TIF_SYSCALL_DOTRACE
366 bne- syscall_dotrace
367 syscall_dotrace_cont:
368 cmplwi 0,r0,NR_syscalls
369 lis r10,sys_call_table@h
370 ori r10,r10,sys_call_table@l
371 slwi r0,r0,2
372 bge- 66f
373
374 barrier_nospec_asm
375
376
377
378
379
380
381
382 lwzx r10,r10,r0
383 mtlr r10
384 addi r9,r1,STACK_FRAME_OVERHEAD
385 PPC440EP_ERR42
386 blrl
387 .globl ret_from_syscall
388 ret_from_syscall:
389 #ifdef CONFIG_DEBUG_RSEQ
390
391 stw r3,GPR3(r1)
392 addi r3,r1,STACK_FRAME_OVERHEAD
393 bl rseq_syscall
394 lwz r3,GPR3(r1)
395 #endif
396 mr r6,r3
397
398 LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
399
400 SYNC
401 MTMSRD(r10)
402 lwz r9,TI_FLAGS(r2)
403 li r8,-MAX_ERRNO
404 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
405 bne- syscall_exit_work
406 cmplw 0,r3,r8
407 blt+ syscall_exit_cont
408 lwz r11,_CCR(r1)
409 neg r3,r3
410 oris r11,r11,0x1000
411 stw r11,_CCR(r1)
412 syscall_exit_cont:
413 lwz r8,_MSR(r1)
414 #ifdef CONFIG_TRACE_IRQFLAGS
415
416
417
418 andi. r10,r8,MSR_EE
419 bne+ 1f
420 stw r3,GPR3(r1)
421 bl trace_hardirqs_off
422 lwz r3,GPR3(r1)
423 1:
424 #endif
425 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
426
427
428 lwz r0,THREAD+THREAD_DBCR0(r2)
429 andis. r10,r0,DBCR0_IDM@h
430 bnel- load_dbcr0
431 #endif
432 #ifdef CONFIG_44x
433 BEGIN_MMU_FTR_SECTION
434 lis r4,icache_44x_need_flush@ha
435 lwz r5,icache_44x_need_flush@l(r4)
436 cmplwi cr0,r5,0
437 bne- 2f
438 1:
439 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_47x)
440 #endif
441 BEGIN_FTR_SECTION
442 lwarx r7,0,r1
443 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
444 stwcx. r0,0,r1
445 ACCOUNT_CPU_USER_EXIT(r2, r5, r7)
446 #ifdef CONFIG_PPC_BOOK3S_32
447 kuep_unlock r5, r7
448 #endif
449 kuap_check r2, r4
450 lwz r4,_LINK(r1)
451 lwz r5,_CCR(r1)
452 mtlr r4
453 mtcr r5
454 lwz r7,_NIP(r1)
455 lwz r2,GPR2(r1)
456 lwz r1,GPR1(r1)
457 #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
458 mtspr SPRN_NRI, r0
459 #endif
460 mtspr SPRN_SRR0,r7
461 mtspr SPRN_SRR1,r8
462 SYNC
463 RFI
464 #ifdef CONFIG_44x
465 2: li r7,0
466 iccci r0,r0
467 stw r7,icache_44x_need_flush@l(r4)
468 b 1b
469 #endif
470
471 66: li r3,-ENOSYS
472 b ret_from_syscall
473
474 .globl ret_from_fork
475 ret_from_fork:
476 REST_NVGPRS(r1)
477 bl schedule_tail
478 li r3,0
479 b ret_from_syscall
480
481 .globl ret_from_kernel_thread
482 ret_from_kernel_thread:
483 REST_NVGPRS(r1)
484 bl schedule_tail
485 mtlr r14
486 mr r3,r15
487 PPC440EP_ERR42
488 blrl
489 li r3,0
490 b ret_from_syscall
491
492
493 syscall_dotrace:
494 SAVE_NVGPRS(r1)
495 li r0,0xc00
496 stw r0,_TRAP(r1)
497 addi r3,r1,STACK_FRAME_OVERHEAD
498 bl do_syscall_trace_enter
499
500
501
502
503
504 mr r0,r3
505 lwz r3,GPR3(r1)
506 lwz r4,GPR4(r1)
507 lwz r5,GPR5(r1)
508 lwz r6,GPR6(r1)
509 lwz r7,GPR7(r1)
510 lwz r8,GPR8(r1)
511 REST_NVGPRS(r1)
512
513 cmplwi r0,NR_syscalls
514
515 bge- ret_from_syscall
516 b syscall_dotrace_cont
517
518 syscall_exit_work:
519 andi. r0,r9,_TIF_RESTOREALL
520 beq+ 0f
521 REST_NVGPRS(r1)
522 b 2f
523 0: cmplw 0,r3,r8
524 blt+ 1f
525 andi. r0,r9,_TIF_NOERROR
526 bne- 1f
527 lwz r11,_CCR(r1)
528 neg r3,r3
529 oris r11,r11,0x1000
530 stw r11,_CCR(r1)
531
532 1: stw r6,RESULT(r1)
533 stw r3,GPR3(r1)
534 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
535 beq 4f
536
537
538
539 li r11,_TIF_PERSYSCALL_MASK
540 addi r12,r2,TI_FLAGS
541 3: lwarx r8,0,r12
542 andc r8,r8,r11
543 #ifdef CONFIG_IBM405_ERR77
544 dcbt 0,r12
545 #endif
546 stwcx. r8,0,r12
547 bne- 3b
548
549 4:
550 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
551 beq ret_from_except
552
553
554
555
556 ori r10,r10,MSR_EE
557 SYNC
558 MTMSRD(r10)
559
560
561 lwz r4,_TRAP(r1)
562 andi. r4,r4,1
563 beq 5f
564 SAVE_NVGPRS(r1)
565 li r4,0xc00
566 stw r4,_TRAP(r1)
567 5:
568 addi r3,r1,STACK_FRAME_OVERHEAD
569 bl do_syscall_trace_leave
570 b ret_from_except_full
571
572
573
574
575
576
577 .globl ppc_fork
578 ppc_fork:
579 SAVE_NVGPRS(r1)
580 lwz r0,_TRAP(r1)
581 rlwinm r0,r0,0,0,30
582 stw r0,_TRAP(r1)
583 b sys_fork
584
585 .globl ppc_vfork
586 ppc_vfork:
587 SAVE_NVGPRS(r1)
588 lwz r0,_TRAP(r1)
589 rlwinm r0,r0,0,0,30
590 stw r0,_TRAP(r1)
591 b sys_vfork
592
593 .globl ppc_clone
594 ppc_clone:
595 SAVE_NVGPRS(r1)
596 lwz r0,_TRAP(r1)
597 rlwinm r0,r0,0,0,30
598 stw r0,_TRAP(r1)
599 b sys_clone
600
601 .globl ppc_clone3
602 ppc_clone3:
603 SAVE_NVGPRS(r1)
604 lwz r0,_TRAP(r1)
605 rlwinm r0,r0,0,0,30
606 stw r0,_TRAP(r1)
607 b sys_clone3
608
609 .globl ppc_swapcontext
610 ppc_swapcontext:
611 SAVE_NVGPRS(r1)
612 lwz r0,_TRAP(r1)
613 rlwinm r0,r0,0,0,30
614 stw r0,_TRAP(r1)
615 b sys_swapcontext
616
617
618
619
620
621
622
623 .globl handle_page_fault
624 handle_page_fault:
625 stw r4,_DAR(r1)
626 addi r3,r1,STACK_FRAME_OVERHEAD
627 #ifdef CONFIG_PPC_BOOK3S_32
628 andis. r0,r5,DSISR_DABRMATCH@h
629 bne- handle_dabr_fault
630 #endif
631 bl do_page_fault
632 cmpwi r3,0
633 beq+ ret_from_except
634 SAVE_NVGPRS(r1)
635 lwz r0,_TRAP(r1)
636 clrrwi r0,r0,1
637 stw r0,_TRAP(r1)
638 mr r5,r3
639 addi r3,r1,STACK_FRAME_OVERHEAD
640 lwz r4,_DAR(r1)
641 bl bad_page_fault
642 b ret_from_except_full
643
644 #ifdef CONFIG_PPC_BOOK3S_32
645
646 handle_dabr_fault:
647 SAVE_NVGPRS(r1)
648 lwz r0,_TRAP(r1)
649 clrrwi r0,r0,1
650 stw r0,_TRAP(r1)
651 bl do_break
652 b ret_from_except_full
653 #endif
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675 _GLOBAL(_switch)
676 stwu r1,-INT_FRAME_SIZE(r1)
677 mflr r0
678 stw r0,INT_FRAME_SIZE+4(r1)
679
680 SAVE_NVGPRS(r1)
681 stw r0,_NIP(r1)
682 mfmsr r11
683 li r0,MSR_FP
684 #ifdef CONFIG_ALTIVEC
685 BEGIN_FTR_SECTION
686 oris r0,r0,MSR_VEC@h
687 mfspr r12,SPRN_VRSAVE
688 stw r12,THREAD+THREAD_VRSAVE(r2)
689 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
690 #endif
691 #ifdef CONFIG_SPE
692 BEGIN_FTR_SECTION
693 oris r0,r0,MSR_SPE@h
694 mfspr r12,SPRN_SPEFSCR
695 stw r12,THREAD+THREAD_SPEFSCR(r2)
696 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
697 #endif
698 and. r0,r0,r11
699 beq+ 1f
700 andc r11,r11,r0
701 MTMSRD(r11)
702 isync
703 1: stw r11,_MSR(r1)
704 mfcr r10
705 stw r10,_CCR(r1)
706 stw r1,KSP(r3)
707
708 kuap_check r2, r0
709 #ifdef CONFIG_SMP
710
711
712
713
714 sync
715 #endif
716
717 tophys(r0,r4)
718 mtspr SPRN_SPRG_THREAD,r0
719 lwz r1,KSP(r4)
720
721
722 mr r3,r2
723 addi r2,r4,-THREAD
724
725 #ifdef CONFIG_ALTIVEC
726 BEGIN_FTR_SECTION
727 lwz r0,THREAD+THREAD_VRSAVE(r2)
728 mtspr SPRN_VRSAVE,r0
729 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
730 #endif
731 #ifdef CONFIG_SPE
732 BEGIN_FTR_SECTION
733 lwz r0,THREAD+THREAD_SPEFSCR(r2)
734 mtspr SPRN_SPEFSCR,r0
735 END_FTR_SECTION_IFSET(CPU_FTR_SPE)
736 #endif
737
738 lwz r0,_CCR(r1)
739 mtcrf 0xFF,r0
740
741 REST_NVGPRS(r1)
742
743 lwz r4,_NIP(r1)
744 mtlr r4
745 addi r1,r1,INT_FRAME_SIZE
746 blr
747
748 .globl fast_exception_return
749 fast_exception_return:
750 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
751 andi. r10,r9,MSR_RI
752 beq 1f
753 #endif
754
755 2: REST_4GPRS(3, r11)
756 lwz r10,_CCR(r11)
757 REST_GPR(1, r11)
758 mtcr r10
759 lwz r10,_LINK(r11)
760 mtlr r10
761
762 li r10, 0
763 stw r10, 8(r11)
764 REST_GPR(10, r11)
765 #if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
766 mtspr SPRN_NRI, r0
767 #endif
768 mtspr SPRN_SRR1,r9
769 mtspr SPRN_SRR0,r12
770 REST_GPR(9, r11)
771 REST_GPR(12, r11)
772 lwz r11,GPR11(r11)
773 SYNC
774 RFI
775
776 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
777
778 1: lis r3,exc_exit_restart_end@ha
779 addi r3,r3,exc_exit_restart_end@l
780 cmplw r12,r3
781 #ifdef CONFIG_PPC_BOOK3S_601
782 bge 2b
783 #else
784 bge 3f
785 #endif
786 lis r4,exc_exit_restart@ha
787 addi r4,r4,exc_exit_restart@l
788 cmplw r12,r4
789 #ifdef CONFIG_PPC_BOOK3S_601
790 blt 2b
791 #else
792 blt 3f
793 #endif
794 lis r3,fee_restarts@ha
795 tophys(r3,r3)
796 lwz r5,fee_restarts@l(r3)
797 addi r5,r5,1
798 stw r5,fee_restarts@l(r3)
799 mr r12,r4
800 b 2b
801
802 .section .bss
803 .align 2
804 fee_restarts:
805 .space 4
806 .previous
807
808
809
810
811 3:
812 li r10,-1
813 stw r10,_TRAP(r11)
814 addi r3,r1,STACK_FRAME_OVERHEAD
815 lis r10,MSR_KERNEL@h
816 ori r10,r10,MSR_KERNEL@l
817 bl transfer_to_handler_full
818 .long unrecoverable_exception
819 .long ret_from_except
820 #endif
821
822 .globl ret_from_except_full
823 ret_from_except_full:
824 REST_NVGPRS(r1)
825
826
827 .globl ret_from_except
828 ret_from_except:
829
830
831
832
833 LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
834 SYNC
835 MTMSRD(r10)
836
837 lwz r3,_MSR(r1)
838 andi. r0,r3,MSR_PR
839 beq resume_kernel
840
841 user_exc_return:
842
843 lwz r9,TI_FLAGS(r2)
844 andi. r0,r9,_TIF_USER_WORK_MASK
845 bne do_work
846
847 restore_user:
848 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
849
850
851 lwz r0,THREAD+THREAD_DBCR0(r2)
852 andis. r10,r0,DBCR0_IDM@h
853 bnel- load_dbcr0
854 #endif
855 ACCOUNT_CPU_USER_EXIT(r2, r10, r11)
856 #ifdef CONFIG_PPC_BOOK3S_32
857 kuep_unlock r10, r11
858 #endif
859
860 b restore
861
862
863 resume_kernel:
864
865 lwz r8,TI_FLAGS(r2)
866 andis. r0,r8,_TIF_EMULATE_STACK_STORE@h
867 beq+ 1f
868
869 addi r8,r1,INT_FRAME_SIZE
870
871 lwz r3,GPR1(r1)
872 subi r3,r3,INT_FRAME_SIZE
873 mr r4,r1
874 mr r1,r3
875
876
877 li r5,INT_FRAME_SIZE/4
878 li r6,0
879 mtctr r5
880 2: lwzx r0,r6,r4
881 stwx r0,r6,r3
882 addi r6,r6,4
883 bdnz 2b
884
885
886 lwz r5,GPR1(r1)
887 stw r8,0(r5)
888
889
890 lis r11,_TIF_EMULATE_STACK_STORE@h
891 addi r5,r2,TI_FLAGS
892 0: lwarx r8,0,r5
893 andc r8,r8,r11
894 #ifdef CONFIG_IBM405_ERR77
895 dcbt 0,r5
896 #endif
897 stwcx. r8,0,r5
898 bne- 0b
899 1:
900
901 #ifdef CONFIG_PREEMPT
902
903 lwz r0,TI_PREEMPT(r2)
904 cmpwi 0,r0,0
905 bne restore_kuap
906 andi. r8,r8,_TIF_NEED_RESCHED
907 beq+ restore_kuap
908 lwz r3,_MSR(r1)
909 andi. r0,r3,MSR_EE
910 beq restore_kuap
911 #ifdef CONFIG_TRACE_IRQFLAGS
912
913
914
915
916 bl trace_hardirqs_off
917 #endif
918 bl preempt_schedule_irq
919 #ifdef CONFIG_TRACE_IRQFLAGS
920
921
922
923 bl trace_hardirqs_on
924 #endif
925 #endif
926 restore_kuap:
927 kuap_restore r1, r2, r9, r10, r0
928
929
930 restore:
931 #ifdef CONFIG_44x
932 BEGIN_MMU_FTR_SECTION
933 b 1f
934 END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_47x)
935 lis r4,icache_44x_need_flush@ha
936 lwz r5,icache_44x_need_flush@l(r4)
937 cmplwi cr0,r5,0
938 beq+ 1f
939 li r6,0
940 iccci r0,r0
941 stw r6,icache_44x_need_flush@l(r4)
942 1:
943 #endif
944
945 lwz r9,_MSR(r1)
946 #ifdef CONFIG_TRACE_IRQFLAGS
947
948
949
950
951
952 andi. r10,r9,MSR_EE
953 beq 1f
954 stwu r1,-32(r1)
955 mflr r0
956 stw r0,4(r1)
957 bl trace_hardirqs_on
958 addi r1, r1, 32
959 lwz r9,_MSR(r1)
960 1:
961 #endif
962
963 lwz r0,GPR0(r1)
964 lwz r2,GPR2(r1)
965 REST_4GPRS(3, r1)
966 REST_2GPRS(7, r1)
967
968 lwz r10,_XER(r1)
969 lwz r11,_CTR(r1)
970 mtspr SPRN_XER,r10
971 mtctr r11
972
973 PPC405_ERR77(0,r1)
974 BEGIN_FTR_SECTION
975 lwarx r11,0,r1
976 END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
977 stwcx. r0,0,r1
978
979 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
980 andi. r10,r9,MSR_RI
981 beql nonrecoverable
982
983 lwz r10,_CCR(r1)
984 lwz r11,_LINK(r1)
985 mtcrf 0xFF,r10
986 mtlr r11
987
988
989 li r10, 0
990 stw r10, 8(r1)
991
992
993
994
995
996
997
998
999
1000 LOAD_REG_IMMEDIATE(r10,MSR_KERNEL & ~MSR_RI)
1001 SYNC
1002 MTMSRD(r10)
1003 .globl exc_exit_restart
1004 exc_exit_restart:
1005 lwz r12,_NIP(r1)
1006 mtspr SPRN_SRR0,r12
1007 mtspr SPRN_SRR1,r9
1008 REST_4GPRS(9, r1)
1009 lwz r1,GPR1(r1)
1010 .globl exc_exit_restart_end
1011 exc_exit_restart_end:
1012 SYNC
1013 RFI
1014
1015 #else
1016
1017
1018
1019
1020
1021
1022
1023 lwz r11,_LINK(r1)
1024 mtlr r11
1025 lwz r10,_CCR(r1)
1026 mtcrf 0xff,r10
1027
1028 li r10, 0
1029 stw r10, 8(r1)
1030 REST_2GPRS(9, r1)
1031 .globl exc_exit_restart
1032 exc_exit_restart:
1033 lwz r11,_NIP(r1)
1034 lwz r12,_MSR(r1)
1035 exc_exit_start:
1036 mtspr SPRN_SRR0,r11
1037 mtspr SPRN_SRR1,r12
1038 REST_2GPRS(11, r1)
1039 lwz r1,GPR1(r1)
1040 .globl exc_exit_restart_end
1041 exc_exit_restart_end:
1042 PPC405_ERR77_SYNC
1043 rfi
1044 b .
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059 #ifdef CONFIG_40x
1060 #define PPC_40x_TURN_OFF_MSR_DR \
1061
1062 \
1063 li r10,MSR_IR; \
1064 mtmsr r10; \
1065 isync; \
1066 tophys(r1, r1);
1067 #else
1068 #define PPC_40x_TURN_OFF_MSR_DR
1069 #endif
1070
1071 #define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi) \
1072 REST_NVGPRS(r1); \
1073 lwz r3,_MSR(r1); \
1074 andi. r3,r3,MSR_PR; \
1075 LOAD_REG_IMMEDIATE(r10,MSR_KERNEL); \
1076 bne user_exc_return; \
1077 lwz r0,GPR0(r1); \
1078 lwz r2,GPR2(r1); \
1079 REST_4GPRS(3, r1); \
1080 REST_2GPRS(7, r1); \
1081 lwz r10,_XER(r1); \
1082 lwz r11,_CTR(r1); \
1083 mtspr SPRN_XER,r10; \
1084 mtctr r11; \
1085 PPC405_ERR77(0,r1); \
1086 stwcx. r0,0,r1; \
1087 lwz r11,_LINK(r1); \
1088 mtlr r11; \
1089 lwz r10,_CCR(r1); \
1090 mtcrf 0xff,r10; \
1091 PPC_40x_TURN_OFF_MSR_DR; \
1092 lwz r9,_DEAR(r1); \
1093 lwz r10,_ESR(r1); \
1094 mtspr SPRN_DEAR,r9; \
1095 mtspr SPRN_ESR,r10; \
1096 lwz r11,_NIP(r1); \
1097 lwz r12,_MSR(r1); \
1098 mtspr exc_lvl_srr0,r11; \
1099 mtspr exc_lvl_srr1,r12; \
1100 lwz r9,GPR9(r1); \
1101 lwz r12,GPR12(r1); \
1102 lwz r10,GPR10(r1); \
1103 lwz r11,GPR11(r1); \
1104 lwz r1,GPR1(r1); \
1105 PPC405_ERR77_SYNC; \
1106 exc_lvl_rfi; \
1107 b .;
1108
1109 #define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1) \
1110 lwz r9,_##exc_lvl_srr0(r1); \
1111 lwz r10,_##exc_lvl_srr1(r1); \
1112 mtspr SPRN_##exc_lvl_srr0,r9; \
1113 mtspr SPRN_##exc_lvl_srr1,r10;
1114
1115 #if defined(CONFIG_PPC_BOOK3E_MMU)
1116 #ifdef CONFIG_PHYS_64BIT
1117 #define RESTORE_MAS7 \
1118 lwz r11,MAS7(r1); \
1119 mtspr SPRN_MAS7,r11;
1120 #else
1121 #define RESTORE_MAS7
1122 #endif
1123 #define RESTORE_MMU_REGS \
1124 lwz r9,MAS0(r1); \
1125 lwz r10,MAS1(r1); \
1126 lwz r11,MAS2(r1); \
1127 mtspr SPRN_MAS0,r9; \
1128 lwz r9,MAS3(r1); \
1129 mtspr SPRN_MAS1,r10; \
1130 lwz r10,MAS6(r1); \
1131 mtspr SPRN_MAS2,r11; \
1132 mtspr SPRN_MAS3,r9; \
1133 mtspr SPRN_MAS6,r10; \
1134 RESTORE_MAS7;
1135 #elif defined(CONFIG_44x)
1136 #define RESTORE_MMU_REGS \
1137 lwz r9,MMUCR(r1); \
1138 mtspr SPRN_MMUCR,r9;
1139 #else
1140 #define RESTORE_MMU_REGS
1141 #endif
1142
1143 #ifdef CONFIG_40x
1144 .globl ret_from_crit_exc
1145 ret_from_crit_exc:
1146 mfspr r9,SPRN_SPRG_THREAD
1147 lis r10,saved_ksp_limit@ha;
1148 lwz r10,saved_ksp_limit@l(r10);
1149 tovirt(r9,r9);
1150 stw r10,KSP_LIMIT(r9)
1151 lis r9,crit_srr0@ha;
1152 lwz r9,crit_srr0@l(r9);
1153 lis r10,crit_srr1@ha;
1154 lwz r10,crit_srr1@l(r10);
1155 mtspr SPRN_SRR0,r9;
1156 mtspr SPRN_SRR1,r10;
1157 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1158 #endif
1159
1160 #ifdef CONFIG_BOOKE
1161 .globl ret_from_crit_exc
1162 ret_from_crit_exc:
1163 mfspr r9,SPRN_SPRG_THREAD
1164 lwz r10,SAVED_KSP_LIMIT(r1)
1165 stw r10,KSP_LIMIT(r9)
1166 RESTORE_xSRR(SRR0,SRR1);
1167 RESTORE_MMU_REGS;
1168 RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
1169
1170 .globl ret_from_debug_exc
1171 ret_from_debug_exc:
1172 mfspr r9,SPRN_SPRG_THREAD
1173 lwz r10,SAVED_KSP_LIMIT(r1)
1174 stw r10,KSP_LIMIT(r9)
1175 RESTORE_xSRR(SRR0,SRR1);
1176 RESTORE_xSRR(CSRR0,CSRR1);
1177 RESTORE_MMU_REGS;
1178 RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
1179
1180 .globl ret_from_mcheck_exc
1181 ret_from_mcheck_exc:
1182 mfspr r9,SPRN_SPRG_THREAD
1183 lwz r10,SAVED_KSP_LIMIT(r1)
1184 stw r10,KSP_LIMIT(r9)
1185 RESTORE_xSRR(SRR0,SRR1);
1186 RESTORE_xSRR(CSRR0,CSRR1);
1187 RESTORE_xSRR(DSRR0,DSRR1);
1188 RESTORE_MMU_REGS;
1189 RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
1190 #endif
1191
1192
1193
1194
1195
1196
1197 load_dbcr0:
1198 mfmsr r10
1199 rlwinm r10,r10,0,~MSR_DE
1200 mtmsr r10
1201 isync
1202 mfspr r10,SPRN_DBCR0
1203 lis r11,global_dbcr0@ha
1204 addi r11,r11,global_dbcr0@l
1205 #ifdef CONFIG_SMP
1206 lwz r9,TASK_CPU(r2)
1207 slwi r9,r9,3
1208 add r11,r11,r9
1209 #endif
1210 stw r10,0(r11)
1211 mtspr SPRN_DBCR0,r0
1212 lwz r10,4(r11)
1213 addi r10,r10,1
1214 stw r10,4(r11)
1215 li r11,-1
1216 mtspr SPRN_DBSR,r11
1217 blr
1218
1219 .section .bss
1220 .align 4
1221 .global global_dbcr0
1222 global_dbcr0:
1223 .space 8*NR_CPUS
1224 .previous
1225 #endif
1226
1227 do_work:
1228 andi. r0,r9,_TIF_NEED_RESCHED
1229 beq do_user_signal
1230
1231 do_resched:
1232 #ifdef CONFIG_TRACE_IRQFLAGS
1233 bl trace_hardirqs_on
1234 mfmsr r10
1235 #endif
1236 ori r10,r10,MSR_EE
1237 SYNC
1238 MTMSRD(r10)
1239 bl schedule
1240 recheck:
1241
1242
1243
1244
1245 LOAD_REG_IMMEDIATE(r10,MSR_KERNEL)
1246 SYNC
1247 MTMSRD(r10)
1248 lwz r9,TI_FLAGS(r2)
1249 andi. r0,r9,_TIF_NEED_RESCHED
1250 bne- do_resched
1251 andi. r0,r9,_TIF_USER_WORK_MASK
1252 beq restore_user
1253 do_user_signal:
1254 ori r10,r10,MSR_EE
1255 SYNC
1256 MTMSRD(r10)
1257
1258 lwz r3,_TRAP(r1)
1259 andi. r0,r3,1
1260 beq 2f
1261 SAVE_NVGPRS(r1)
1262 rlwinm r3,r3,0,0,30
1263 stw r3,_TRAP(r1)
1264 2: addi r3,r1,STACK_FRAME_OVERHEAD
1265 mr r4,r9
1266 bl do_notify_resume
1267 REST_NVGPRS(r1)
1268 b recheck
1269
1270
1271
1272
1273
1274
1275 nonrecoverable:
1276 lis r10,exc_exit_restart_end@ha
1277 addi r10,r10,exc_exit_restart_end@l
1278 cmplw r12,r10
1279 #ifdef CONFIG_PPC_BOOK3S_601
1280 bgelr
1281 #else
1282 bge 3f
1283 #endif
1284 lis r11,exc_exit_restart@ha
1285 addi r11,r11,exc_exit_restart@l
1286 cmplw r12,r11
1287 #ifdef CONFIG_PPC_BOOK3S_601
1288 bltlr
1289 #else
1290 blt 3f
1291 #endif
1292 lis r10,ee_restarts@ha
1293 lwz r12,ee_restarts@l(r10)
1294 addi r12,r12,1
1295 stw r12,ee_restarts@l(r10)
1296 mr r12,r11
1297 blr
1298 3:
1299
1300 lwz r3,_TRAP(r1)
1301 andi. r0,r3,1
1302 beq 5f
1303 SAVE_NVGPRS(r1)
1304 rlwinm r3,r3,0,0,30
1305 stw r3,_TRAP(r1)
1306 5: mfspr r2,SPRN_SPRG_THREAD
1307 addi r2,r2,-THREAD
1308 tovirt(r2,r2)
1309 4: addi r3,r1,STACK_FRAME_OVERHEAD
1310 bl unrecoverable_exception
1311
1312 b 4b
1313
1314 .section .bss
1315 .align 2
1316 ee_restarts:
1317 .space 4
1318 .previous
1319
1320
1321
1322
1323
1324
1325 #ifdef CONFIG_PPC_RTAS
1326
1327
1328
1329
1330 _GLOBAL(enter_rtas)
1331 stwu r1,-INT_FRAME_SIZE(r1)
1332 mflr r0
1333 stw r0,INT_FRAME_SIZE+4(r1)
1334 LOAD_REG_ADDR(r4, rtas)
1335 lis r6,1f@ha
1336 addi r6,r6,1f@l
1337 tophys(r6,r6)
1338 tophys(r7,r1)
1339 lwz r8,RTASENTRY(r4)
1340 lwz r4,RTASBASE(r4)
1341 mfmsr r9
1342 stw r9,8(r1)
1343 LOAD_REG_IMMEDIATE(r0,MSR_KERNEL)
1344 SYNC
1345 MTMSRD(r0)
1346 li r9,MSR_KERNEL & ~(MSR_IR|MSR_DR)
1347 mtlr r6
1348 stw r7, THREAD + RTAS_SP(r2)
1349 mtspr SPRN_SRR0,r8
1350 mtspr SPRN_SRR1,r9
1351 RFI
1352 1: tophys(r9,r1)
1353 lwz r8,INT_FRAME_SIZE+4(r9)
1354 lwz r9,8(r9)
1355 addi r1,r1,INT_FRAME_SIZE
1356 li r0,0
1357 tophys(r7, r2)
1358 stw r0, THREAD + RTAS_SP(r7)
1359 mtspr SPRN_SRR0,r8
1360 mtspr SPRN_SRR1,r9
1361 RFI
1362
1363 .globl machine_check_in_rtas
1364 machine_check_in_rtas:
1365 twi 31,0,0
1366
1367
1368 #endif