1/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
2 *
3 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
4 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
5 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
6 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
7 */
8
9#include <asm/head.h>
10#include <asm/asi.h>
11#include <asm/page.h>
12#include <asm/pgtable.h>
13#include <asm/tsb.h>
14
15	.text
16	.align		32
17
18kvmap_itlb:
19	/* g6: TAG TARGET */
20	mov		TLB_TAG_ACCESS, %g4
21	ldxa		[%g4] ASI_IMMU, %g4
22
23	/* sun4v_itlb_miss branches here with the missing virtual
24	 * address already loaded into %g4
25	 */
26kvmap_itlb_4v:
27
28	/* Catch kernel NULL pointer calls.  */
29	sethi		%hi(PAGE_SIZE), %g5
30	cmp		%g4, %g5
31	blu,pn		%xcc, kvmap_itlb_longpath
32	 nop
33
34	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
35
36kvmap_itlb_tsb_miss:
37	sethi		%hi(LOW_OBP_ADDRESS), %g5
38	cmp		%g4, %g5
39	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
40	 mov		0x1, %g5
41	sllx		%g5, 32, %g5
42	cmp		%g4, %g5
43	blu,pn		%xcc, kvmap_itlb_obp
44	 nop
45
46kvmap_itlb_vmalloc_addr:
47	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
48
49	TSB_LOCK_TAG(%g1, %g2, %g7)
50	TSB_WRITE(%g1, %g5, %g6)
51
52	/* fallthrough to TLB load */
53
54kvmap_itlb_load:
55
56661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
57	retry
58	.section	.sun4v_2insn_patch, "ax"
59	.word		661b
60	nop
61	nop
62	.previous
63
64	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
65	 * instruction get nop'd out and we get here to branch
66	 * to the sun4v tlb load code.  The registers are setup
67	 * as follows:
68	 *
69	 * %g4: vaddr
70	 * %g5: PTE
71	 * %g6:	TAG
72	 *
73	 * The sun4v TLB load wants the PTE in %g3 so we fix that
74	 * up here.
75	 */
76	ba,pt		%xcc, sun4v_itlb_load
77	 mov		%g5, %g3
78
79kvmap_itlb_longpath:
80
81661:	rdpr	%pstate, %g5
82	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
83	.section .sun4v_2insn_patch, "ax"
84	.word	661b
85	SET_GL(1)
86	nop
87	.previous
88
89	rdpr	%tpc, %g5
90	ba,pt	%xcc, sparc64_realfault_common
91	 mov	FAULT_CODE_ITLB, %g4
92
93kvmap_itlb_obp:
94	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
95
96	TSB_LOCK_TAG(%g1, %g2, %g7)
97
98	TSB_WRITE(%g1, %g5, %g6)
99
100	ba,pt		%xcc, kvmap_itlb_load
101	 nop
102
103kvmap_dtlb_obp:
104	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
105
106	TSB_LOCK_TAG(%g1, %g2, %g7)
107
108	TSB_WRITE(%g1, %g5, %g6)
109
110	ba,pt		%xcc, kvmap_dtlb_load
111	 nop
112
113kvmap_linear_early:
114	sethi		%hi(kern_linear_pte_xor), %g7
115	ldx		[%g7 + %lo(kern_linear_pte_xor)], %g2
116	ba,pt		%xcc, kvmap_dtlb_tsb4m_load
117	 xor		%g2, %g4, %g5
118
119	.align		32
120kvmap_dtlb_tsb4m_load:
121	TSB_LOCK_TAG(%g1, %g2, %g7)
122	TSB_WRITE(%g1, %g5, %g6)
123	ba,pt		%xcc, kvmap_dtlb_load
124	 nop
125
126kvmap_dtlb:
127	/* %g6: TAG TARGET */
128	mov		TLB_TAG_ACCESS, %g4
129	ldxa		[%g4] ASI_DMMU, %g4
130
131	/* sun4v_dtlb_miss branches here with the missing virtual
132	 * address already loaded into %g4
133	 */
134kvmap_dtlb_4v:
135	brgez,pn	%g4, kvmap_dtlb_nonlinear
136	 nop
137
138#ifdef CONFIG_DEBUG_PAGEALLOC
139	/* Index through the base page size TSB even for linear
140	 * mappings when using page allocation debugging.
141	 */
142	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
143#else
144	/* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
145	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
146#endif
147	/* Linear mapping TSB lookup failed.  Fallthrough to kernel
148	 * page table based lookup.
149	 */
150	.globl		kvmap_linear_patch
151kvmap_linear_patch:
152	ba,a,pt		%xcc, kvmap_linear_early
153
154kvmap_dtlb_vmalloc_addr:
155	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
156
157	TSB_LOCK_TAG(%g1, %g2, %g7)
158	TSB_WRITE(%g1, %g5, %g6)
159
160	/* fallthrough to TLB load */
161
162kvmap_dtlb_load:
163
164661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
165	retry
166	.section	.sun4v_2insn_patch, "ax"
167	.word		661b
168	nop
169	nop
170	.previous
171
172	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
173	 * instruction get nop'd out and we get here to branch
174	 * to the sun4v tlb load code.  The registers are setup
175	 * as follows:
176	 *
177	 * %g4: vaddr
178	 * %g5: PTE
179	 * %g6:	TAG
180	 *
181	 * The sun4v TLB load wants the PTE in %g3 so we fix that
182	 * up here.
183	 */
184	ba,pt		%xcc, sun4v_dtlb_load
185	 mov		%g5, %g3
186
187#ifdef CONFIG_SPARSEMEM_VMEMMAP
188kvmap_vmemmap:
189	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
190	ba,a,pt		%xcc, kvmap_dtlb_load
191#endif
192
193kvmap_dtlb_nonlinear:
194	/* Catch kernel NULL pointer derefs.  */
195	sethi		%hi(PAGE_SIZE), %g5
196	cmp		%g4, %g5
197	bleu,pn		%xcc, kvmap_dtlb_longpath
198	 nop
199
200#ifdef CONFIG_SPARSEMEM_VMEMMAP
201	/* Do not use the TSB for vmemmap.  */
202	sethi		%hi(VMEMMAP_BASE), %g5
203	ldx		[%g5 + %lo(VMEMMAP_BASE)], %g5
204	cmp		%g4,%g5
205	bgeu,pn		%xcc, kvmap_vmemmap
206	 nop
207#endif
208
209	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
210
211kvmap_dtlb_tsbmiss:
212	sethi		%hi(MODULES_VADDR), %g5
213	cmp		%g4, %g5
214	blu,pn		%xcc, kvmap_dtlb_longpath
215	 sethi		%hi(VMALLOC_END), %g5
216	ldx		[%g5 + %lo(VMALLOC_END)], %g5
217	cmp		%g4, %g5
218	bgeu,pn		%xcc, kvmap_dtlb_longpath
219	 nop
220
221kvmap_check_obp:
222	sethi		%hi(LOW_OBP_ADDRESS), %g5
223	cmp		%g4, %g5
224	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
225	 mov		0x1, %g5
226	sllx		%g5, 32, %g5
227	cmp		%g4, %g5
228	blu,pn		%xcc, kvmap_dtlb_obp
229	 nop
230	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
231	 nop
232
233kvmap_dtlb_longpath:
234
235661:	rdpr	%pstate, %g5
236	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
237	.section .sun4v_2insn_patch, "ax"
238	.word	661b
239	SET_GL(1)
240	ldxa		[%g0] ASI_SCRATCHPAD, %g5
241	.previous
242
243	rdpr	%tl, %g3
244	cmp	%g3, 1
245
246661:	mov	TLB_TAG_ACCESS, %g4
247	ldxa	[%g4] ASI_DMMU, %g5
248	.section .sun4v_2insn_patch, "ax"
249	.word	661b
250	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
251	nop
252	.previous
253
254	be,pt	%xcc, sparc64_realfault_common
255	 mov	FAULT_CODE_DTLB, %g4
256	ba,pt	%xcc, winfix_trampoline
257	 nop
258