This source file includes following definitions.
- kernel_fpu_disabled
- interrupted_kernel_fpu_idle
- interrupted_user_mode
- irq_fpu_usable
- kernel_fpu_begin
- kernel_fpu_end
- fpu__save
- fpstate_init_fstate
- fpstate_init
- fpu__copy
- fpu__initialize
- fpu__prepare_read
- fpu__prepare_write
- fpu__drop
- copy_init_fpstate_to_fpregs
- fpu__clear
- switch_fpu_return
- fpregs_assert_state_consistent
- fpregs_mark_activate
- fpu__exception_code
1
2
3
4
5
6
7
8
9 #include <asm/fpu/internal.h>
10 #include <asm/fpu/regset.h>
11 #include <asm/fpu/signal.h>
12 #include <asm/fpu/types.h>
13 #include <asm/traps.h>
14 #include <asm/irq_regs.h>
15
16 #include <linux/hardirq.h>
17 #include <linux/pkeys.h>
18
19 #define CREATE_TRACE_POINTS
20 #include <asm/trace/fpu.h>
21
22
23
24
25
26 union fpregs_state init_fpstate __read_mostly;
27
28
29
30
31
32
33
34
35
36
37
38
39 static DEFINE_PER_CPU(bool, in_kernel_fpu);
40
41
42
43
44 DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
45
46 static bool kernel_fpu_disabled(void)
47 {
48 return this_cpu_read(in_kernel_fpu);
49 }
50
51 static bool interrupted_kernel_fpu_idle(void)
52 {
53 return !kernel_fpu_disabled();
54 }
55
56
57
58
59
60
61
62
63
64 static bool interrupted_user_mode(void)
65 {
66 struct pt_regs *regs = get_irq_regs();
67 return regs && user_mode(regs);
68 }
69
70
71
72
73
74
75
76
77 bool irq_fpu_usable(void)
78 {
79 return !in_interrupt() ||
80 interrupted_user_mode() ||
81 interrupted_kernel_fpu_idle();
82 }
83 EXPORT_SYMBOL(irq_fpu_usable);
84
85 void kernel_fpu_begin(void)
86 {
87 preempt_disable();
88
89 WARN_ON_FPU(!irq_fpu_usable());
90 WARN_ON_FPU(this_cpu_read(in_kernel_fpu));
91
92 this_cpu_write(in_kernel_fpu, true);
93
94 if (!(current->flags & PF_KTHREAD) &&
95 !test_thread_flag(TIF_NEED_FPU_LOAD)) {
96 set_thread_flag(TIF_NEED_FPU_LOAD);
97
98
99
100
101 copy_fpregs_to_fpstate(¤t->thread.fpu);
102 }
103 __cpu_invalidate_fpregs_state();
104 }
105 EXPORT_SYMBOL_GPL(kernel_fpu_begin);
106
107 void kernel_fpu_end(void)
108 {
109 WARN_ON_FPU(!this_cpu_read(in_kernel_fpu));
110
111 this_cpu_write(in_kernel_fpu, false);
112 preempt_enable();
113 }
114 EXPORT_SYMBOL_GPL(kernel_fpu_end);
115
116
117
118
119
120
121 void fpu__save(struct fpu *fpu)
122 {
123 WARN_ON_FPU(fpu != ¤t->thread.fpu);
124
125 fpregs_lock();
126 trace_x86_fpu_before_save(fpu);
127
128 if (!test_thread_flag(TIF_NEED_FPU_LOAD)) {
129 if (!copy_fpregs_to_fpstate(fpu)) {
130 copy_kernel_to_fpregs(&fpu->state);
131 }
132 }
133
134 trace_x86_fpu_after_save(fpu);
135 fpregs_unlock();
136 }
137
138
139
140
141 static inline void fpstate_init_fstate(struct fregs_state *fp)
142 {
143 fp->cwd = 0xffff037fu;
144 fp->swd = 0xffff0000u;
145 fp->twd = 0xffffffffu;
146 fp->fos = 0xffff0000u;
147 }
148
149 void fpstate_init(union fpregs_state *state)
150 {
151 if (!static_cpu_has(X86_FEATURE_FPU)) {
152 fpstate_init_soft(&state->soft);
153 return;
154 }
155
156 memset(state, 0, fpu_kernel_xstate_size);
157
158 if (static_cpu_has(X86_FEATURE_XSAVES))
159 fpstate_init_xstate(&state->xsave);
160 if (static_cpu_has(X86_FEATURE_FXSR))
161 fpstate_init_fxstate(&state->fxsave);
162 else
163 fpstate_init_fstate(&state->fsave);
164 }
165 EXPORT_SYMBOL_GPL(fpstate_init);
166
167 int fpu__copy(struct task_struct *dst, struct task_struct *src)
168 {
169 struct fpu *dst_fpu = &dst->thread.fpu;
170 struct fpu *src_fpu = &src->thread.fpu;
171
172 dst_fpu->last_cpu = -1;
173
174 if (!static_cpu_has(X86_FEATURE_FPU))
175 return 0;
176
177 WARN_ON_FPU(src_fpu != ¤t->thread.fpu);
178
179
180
181
182
183 memset(&dst_fpu->state.xsave, 0, fpu_kernel_xstate_size);
184
185
186
187
188
189
190
191
192
193 fpregs_lock();
194 if (test_thread_flag(TIF_NEED_FPU_LOAD))
195 memcpy(&dst_fpu->state, &src_fpu->state, fpu_kernel_xstate_size);
196
197 else if (!copy_fpregs_to_fpstate(dst_fpu))
198 copy_kernel_to_fpregs(&dst_fpu->state);
199
200 fpregs_unlock();
201
202 set_tsk_thread_flag(dst, TIF_NEED_FPU_LOAD);
203
204 trace_x86_fpu_copy_src(src_fpu);
205 trace_x86_fpu_copy_dst(dst_fpu);
206
207 return 0;
208 }
209
210
211
212
213
214 static void fpu__initialize(struct fpu *fpu)
215 {
216 WARN_ON_FPU(fpu != ¤t->thread.fpu);
217
218 set_thread_flag(TIF_NEED_FPU_LOAD);
219 fpstate_init(&fpu->state);
220 trace_x86_fpu_init_state(fpu);
221 }
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237 void fpu__prepare_read(struct fpu *fpu)
238 {
239 if (fpu == ¤t->thread.fpu)
240 fpu__save(fpu);
241 }
242
243
244
245
246
247
248
249
250
251
252
253
254
255 void fpu__prepare_write(struct fpu *fpu)
256 {
257
258
259
260
261 WARN_ON_FPU(fpu == ¤t->thread.fpu);
262
263
264 __fpu_invalidate_fpregs_state(fpu);
265 }
266
267
268
269
270
271
272
273
274
275
276 void fpu__drop(struct fpu *fpu)
277 {
278 preempt_disable();
279
280 if (fpu == ¤t->thread.fpu) {
281
282 asm volatile("1: fwait\n"
283 "2:\n"
284 _ASM_EXTABLE(1b, 2b));
285 fpregs_deactivate(fpu);
286 }
287
288 trace_x86_fpu_dropped(fpu);
289
290 preempt_enable();
291 }
292
293
294
295
296
297 static inline void copy_init_fpstate_to_fpregs(void)
298 {
299 fpregs_lock();
300
301 if (use_xsave())
302 copy_kernel_to_xregs(&init_fpstate.xsave, -1);
303 else if (static_cpu_has(X86_FEATURE_FXSR))
304 copy_kernel_to_fxregs(&init_fpstate.fxsave);
305 else
306 copy_kernel_to_fregs(&init_fpstate.fsave);
307
308 if (boot_cpu_has(X86_FEATURE_OSPKE))
309 copy_init_pkru_to_fpregs();
310
311 fpregs_mark_activate();
312 fpregs_unlock();
313 }
314
315
316
317
318
319
320
321 void fpu__clear(struct fpu *fpu)
322 {
323 WARN_ON_FPU(fpu != ¤t->thread.fpu);
324
325 fpu__drop(fpu);
326
327
328
329
330 fpu__initialize(fpu);
331 if (static_cpu_has(X86_FEATURE_FPU))
332 copy_init_fpstate_to_fpregs();
333 }
334
335
336
337
338 void switch_fpu_return(void)
339 {
340 if (!static_cpu_has(X86_FEATURE_FPU))
341 return;
342
343 __fpregs_load_activate();
344 }
345 EXPORT_SYMBOL_GPL(switch_fpu_return);
346
347 #ifdef CONFIG_X86_DEBUG_FPU
348
349
350
351
352
353 void fpregs_assert_state_consistent(void)
354 {
355 struct fpu *fpu = ¤t->thread.fpu;
356
357 if (test_thread_flag(TIF_NEED_FPU_LOAD))
358 return;
359
360 WARN_ON_FPU(!fpregs_state_valid(fpu, smp_processor_id()));
361 }
362 EXPORT_SYMBOL_GPL(fpregs_assert_state_consistent);
363 #endif
364
365 void fpregs_mark_activate(void)
366 {
367 struct fpu *fpu = ¤t->thread.fpu;
368
369 fpregs_activate(fpu);
370 fpu->last_cpu = smp_processor_id();
371 clear_thread_flag(TIF_NEED_FPU_LOAD);
372 }
373 EXPORT_SYMBOL_GPL(fpregs_mark_activate);
374
375
376
377
378
379 int fpu__exception_code(struct fpu *fpu, int trap_nr)
380 {
381 int err;
382
383 if (trap_nr == X86_TRAP_MF) {
384 unsigned short cwd, swd;
385
386
387
388
389
390
391
392
393
394
395 if (boot_cpu_has(X86_FEATURE_FXSR)) {
396 cwd = fpu->state.fxsave.cwd;
397 swd = fpu->state.fxsave.swd;
398 } else {
399 cwd = (unsigned short)fpu->state.fsave.cwd;
400 swd = (unsigned short)fpu->state.fsave.swd;
401 }
402
403 err = swd & ~cwd;
404 } else {
405
406
407
408
409
410
411 unsigned short mxcsr = MXCSR_DEFAULT;
412
413 if (boot_cpu_has(X86_FEATURE_XMM))
414 mxcsr = fpu->state.fxsave.mxcsr;
415
416 err = ~(mxcsr >> 7) & mxcsr;
417 }
418
419 if (err & 0x001) {
420
421
422
423
424
425 return FPE_FLTINV;
426 } else if (err & 0x004) {
427 return FPE_FLTDIV;
428 } else if (err & 0x008) {
429 return FPE_FLTOVF;
430 } else if (err & 0x012) {
431 return FPE_FLTUND;
432 } else if (err & 0x020) {
433 return FPE_FLTRES;
434 }
435
436
437
438
439
440
441 return 0;
442 }