1
2
3
4
5
6
7 #include <linux/init.h>
8 #include <linux/linkage.h>
9
10 #include <asm/asm.h>
11 #include <asm/csr.h>
12 #include <asm/unistd.h>
13 #include <asm/thread_info.h>
14 #include <asm/asm-offsets.h>
15
16 .text
17 .altmacro
18
19
20
21
22
23 .macro SAVE_ALL
24 LOCAL _restore_kernel_tpsp
25 LOCAL _save_context
26
27
28
29
30
31
32 csrrw tp, CSR_SSCRATCH, tp
33 bnez tp, _save_context
34
35 _restore_kernel_tpsp:
36 csrr tp, CSR_SSCRATCH
37 REG_S sp, TASK_TI_KERNEL_SP(tp)
38 _save_context:
39 REG_S sp, TASK_TI_USER_SP(tp)
40 REG_L sp, TASK_TI_KERNEL_SP(tp)
41 addi sp, sp, -(PT_SIZE_ON_STACK)
42 REG_S x1, PT_RA(sp)
43 REG_S x3, PT_GP(sp)
44 REG_S x5, PT_T0(sp)
45 REG_S x6, PT_T1(sp)
46 REG_S x7, PT_T2(sp)
47 REG_S x8, PT_S0(sp)
48 REG_S x9, PT_S1(sp)
49 REG_S x10, PT_A0(sp)
50 REG_S x11, PT_A1(sp)
51 REG_S x12, PT_A2(sp)
52 REG_S x13, PT_A3(sp)
53 REG_S x14, PT_A4(sp)
54 REG_S x15, PT_A5(sp)
55 REG_S x16, PT_A6(sp)
56 REG_S x17, PT_A7(sp)
57 REG_S x18, PT_S2(sp)
58 REG_S x19, PT_S3(sp)
59 REG_S x20, PT_S4(sp)
60 REG_S x21, PT_S5(sp)
61 REG_S x22, PT_S6(sp)
62 REG_S x23, PT_S7(sp)
63 REG_S x24, PT_S8(sp)
64 REG_S x25, PT_S9(sp)
65 REG_S x26, PT_S10(sp)
66 REG_S x27, PT_S11(sp)
67 REG_S x28, PT_T3(sp)
68 REG_S x29, PT_T4(sp)
69 REG_S x30, PT_T5(sp)
70 REG_S x31, PT_T6(sp)
71
72
73
74
75
76
77
78
79 li t0, SR_SUM | SR_FS
80
81 REG_L s0, TASK_TI_USER_SP(tp)
82 csrrc s1, CSR_SSTATUS, t0
83 csrr s2, CSR_SEPC
84 csrr s3, CSR_STVAL
85 csrr s4, CSR_SCAUSE
86 csrr s5, CSR_SSCRATCH
87 REG_S s0, PT_SP(sp)
88 REG_S s1, PT_SSTATUS(sp)
89 REG_S s2, PT_SEPC(sp)
90 REG_S s3, PT_SBADADDR(sp)
91 REG_S s4, PT_SCAUSE(sp)
92 REG_S s5, PT_TP(sp)
93 .endm
94
95
96
97
98
99 .macro RESTORE_ALL
100 REG_L a0, PT_SSTATUS(sp)
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118 REG_L a2, PT_SEPC(sp)
119 REG_SC x0, a2, PT_SEPC(sp)
120
121 csrw CSR_SSTATUS, a0
122 csrw CSR_SEPC, a2
123
124 REG_L x1, PT_RA(sp)
125 REG_L x3, PT_GP(sp)
126 REG_L x4, PT_TP(sp)
127 REG_L x5, PT_T0(sp)
128 REG_L x6, PT_T1(sp)
129 REG_L x7, PT_T2(sp)
130 REG_L x8, PT_S0(sp)
131 REG_L x9, PT_S1(sp)
132 REG_L x10, PT_A0(sp)
133 REG_L x11, PT_A1(sp)
134 REG_L x12, PT_A2(sp)
135 REG_L x13, PT_A3(sp)
136 REG_L x14, PT_A4(sp)
137 REG_L x15, PT_A5(sp)
138 REG_L x16, PT_A6(sp)
139 REG_L x17, PT_A7(sp)
140 REG_L x18, PT_S2(sp)
141 REG_L x19, PT_S3(sp)
142 REG_L x20, PT_S4(sp)
143 REG_L x21, PT_S5(sp)
144 REG_L x22, PT_S6(sp)
145 REG_L x23, PT_S7(sp)
146 REG_L x24, PT_S8(sp)
147 REG_L x25, PT_S9(sp)
148 REG_L x26, PT_S10(sp)
149 REG_L x27, PT_S11(sp)
150 REG_L x28, PT_T3(sp)
151 REG_L x29, PT_T4(sp)
152 REG_L x30, PT_T5(sp)
153 REG_L x31, PT_T6(sp)
154
155 REG_L x2, PT_SP(sp)
156 .endm
157
158 #if !IS_ENABLED(CONFIG_PREEMPT)
159 .set resume_kernel, restore_all
160 #endif
161
162 ENTRY(handle_exception)
163 SAVE_ALL
164
165
166
167
168
169 csrw CSR_SSCRATCH, x0
170
171
172 .option push
173 .option norelax
174 la gp, __global_pointer$
175 .option pop
176
177 la ra, ret_from_exception
178
179
180
181
182 bge s4, zero, 1f
183
184
185 move a0, sp
186 tail do_IRQ
187 1:
188
189
190 andi t0, s1, SR_SPIE
191 beqz t0, 1f
192 csrs CSR_SSTATUS, SR_SIE
193
194 1:
195
196 li t0, EXC_SYSCALL
197 beq s4, t0, handle_syscall
198
199
200 slli t0, s4, RISCV_LGPTR
201 la t1, excp_vect_table
202 la t2, excp_vect_table_end
203 move a0, sp
204 add t0, t1, t0
205
206 bgeu t0, t2, 1f
207 REG_L t0, 0(t0)
208 jr t0
209 1:
210 tail do_trap_unknown
211
212 handle_syscall:
213
214 REG_S a0, PT_ORIG_A0(sp)
215
216
217
218
219 addi s2, s2, 0x4
220 REG_S s2, PT_SEPC(sp)
221
222 REG_L t0, TASK_TI_FLAGS(tp)
223 andi t0, t0, _TIF_SYSCALL_WORK
224 bnez t0, handle_syscall_trace_enter
225 check_syscall_nr:
226
227 li t0, __NR_syscalls
228 la s0, sys_ni_syscall
229
230 bgeu a7, t0, 1f
231 la s0, sys_call_table
232 slli t0, a7, RISCV_LGPTR
233 add s0, s0, t0
234 REG_L s0, 0(s0)
235 1:
236 jalr s0
237
238 ret_from_syscall:
239
240 REG_S a0, PT_A0(sp)
241
242 REG_L t0, TASK_TI_FLAGS(tp)
243 andi t0, t0, _TIF_SYSCALL_WORK
244 bnez t0, handle_syscall_trace_exit
245
246 ret_from_exception:
247 REG_L s0, PT_SSTATUS(sp)
248 csrc CSR_SSTATUS, SR_SIE
249 andi s0, s0, SR_SPP
250 bnez s0, resume_kernel
251
252 resume_userspace:
253
254 REG_L s0, TASK_TI_FLAGS(tp)
255 andi s1, s0, _TIF_WORK_MASK
256 bnez s1, work_pending
257
258
259 addi s0, sp, PT_SIZE_ON_STACK
260 REG_S s0, TASK_TI_KERNEL_SP(tp)
261
262
263
264
265
266 csrw CSR_SSCRATCH, tp
267
268 restore_all:
269 RESTORE_ALL
270 sret
271
272 #if IS_ENABLED(CONFIG_PREEMPT)
273 resume_kernel:
274 REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
275 bnez s0, restore_all
276 REG_L s0, TASK_TI_FLAGS(tp)
277 andi s0, s0, _TIF_NEED_RESCHED
278 beqz s0, restore_all
279 call preempt_schedule_irq
280 j restore_all
281 #endif
282
283 work_pending:
284
285 la ra, ret_from_exception
286 andi s1, s0, _TIF_NEED_RESCHED
287 bnez s1, work_resched
288 work_notifysig:
289
290 csrs CSR_SSTATUS, SR_SIE
291 move a0, sp
292 move a1, s0
293 tail do_notify_resume
294 work_resched:
295 tail schedule
296
297
298 handle_syscall_trace_enter:
299 move a0, sp
300 call do_syscall_trace_enter
301 REG_L a0, PT_A0(sp)
302 REG_L a1, PT_A1(sp)
303 REG_L a2, PT_A2(sp)
304 REG_L a3, PT_A3(sp)
305 REG_L a4, PT_A4(sp)
306 REG_L a5, PT_A5(sp)
307 REG_L a6, PT_A6(sp)
308 REG_L a7, PT_A7(sp)
309 j check_syscall_nr
310 handle_syscall_trace_exit:
311 move a0, sp
312 call do_syscall_trace_exit
313 j ret_from_exception
314
315 END(handle_exception)
316
317 ENTRY(ret_from_fork)
318 la ra, ret_from_exception
319 tail schedule_tail
320 ENDPROC(ret_from_fork)
321
322 ENTRY(ret_from_kernel_thread)
323 call schedule_tail
324
325 la ra, ret_from_exception
326 move a0, s1
327 jr s0
328 ENDPROC(ret_from_kernel_thread)
329
330
331
332
333
334
335
336
337
338
339
340
341 ENTRY(__switch_to)
342
343 li a4, TASK_THREAD_RA
344 add a3, a0, a4
345 add a4, a1, a4
346 REG_S ra, TASK_THREAD_RA_RA(a3)
347 REG_S sp, TASK_THREAD_SP_RA(a3)
348 REG_S s0, TASK_THREAD_S0_RA(a3)
349 REG_S s1, TASK_THREAD_S1_RA(a3)
350 REG_S s2, TASK_THREAD_S2_RA(a3)
351 REG_S s3, TASK_THREAD_S3_RA(a3)
352 REG_S s4, TASK_THREAD_S4_RA(a3)
353 REG_S s5, TASK_THREAD_S5_RA(a3)
354 REG_S s6, TASK_THREAD_S6_RA(a3)
355 REG_S s7, TASK_THREAD_S7_RA(a3)
356 REG_S s8, TASK_THREAD_S8_RA(a3)
357 REG_S s9, TASK_THREAD_S9_RA(a3)
358 REG_S s10, TASK_THREAD_S10_RA(a3)
359 REG_S s11, TASK_THREAD_S11_RA(a3)
360
361 REG_L ra, TASK_THREAD_RA_RA(a4)
362 REG_L sp, TASK_THREAD_SP_RA(a4)
363 REG_L s0, TASK_THREAD_S0_RA(a4)
364 REG_L s1, TASK_THREAD_S1_RA(a4)
365 REG_L s2, TASK_THREAD_S2_RA(a4)
366 REG_L s3, TASK_THREAD_S3_RA(a4)
367 REG_L s4, TASK_THREAD_S4_RA(a4)
368 REG_L s5, TASK_THREAD_S5_RA(a4)
369 REG_L s6, TASK_THREAD_S6_RA(a4)
370 REG_L s7, TASK_THREAD_S7_RA(a4)
371 REG_L s8, TASK_THREAD_S8_RA(a4)
372 REG_L s9, TASK_THREAD_S9_RA(a4)
373 REG_L s10, TASK_THREAD_S10_RA(a4)
374 REG_L s11, TASK_THREAD_S11_RA(a4)
375
376 lw a3, TASK_TI_CPU(a0)
377 lw a4, TASK_TI_CPU(a1)
378 sw a3, TASK_TI_CPU(a1)
379 sw a4, TASK_TI_CPU(a0)
380 #if TASK_TI != 0
381 #error "TASK_TI != 0: tp will contain a 'struct thread_info', not a 'struct task_struct' so get_current() won't work."
382 addi tp, a1, TASK_TI
383 #else
384 move tp, a1
385 #endif
386 ret
387 ENDPROC(__switch_to)
388
389 .section ".rodata"
390
391 ENTRY(excp_vect_table)
392 RISCV_PTR do_trap_insn_misaligned
393 RISCV_PTR do_trap_insn_fault
394 RISCV_PTR do_trap_insn_illegal
395 RISCV_PTR do_trap_break
396 RISCV_PTR do_trap_load_misaligned
397 RISCV_PTR do_trap_load_fault
398 RISCV_PTR do_trap_store_misaligned
399 RISCV_PTR do_trap_store_fault
400 RISCV_PTR do_trap_ecall_u
401 RISCV_PTR do_trap_ecall_s
402 RISCV_PTR do_trap_unknown
403 RISCV_PTR do_trap_ecall_m
404 RISCV_PTR do_page_fault
405 RISCV_PTR do_page_fault
406 RISCV_PTR do_trap_unknown
407 RISCV_PTR do_page_fault
408 excp_vect_table_end:
409 END(excp_vect_table)