This source file includes following definitions.
- fpu__init_cpu_generic
- fpu__init_cpu
- fpu__probe_without_cpuid
- fpu__init_system_early_generic
- fpu__init_system_mxcsr
- fpu__init_system_generic
- fpu__init_task_struct_size
- fpu__init_system_xstate_size_legacy
- fpu__get_supported_xfeatures_mask
- fpu__init_system_ctx_switch
- fpu__init_parse_early_param
- fpu__init_system
1
2
3
4
5 #include <asm/fpu/internal.h>
6 #include <asm/tlbflush.h>
7 #include <asm/setup.h>
8 #include <asm/cmdline.h>
9
10 #include <linux/sched.h>
11 #include <linux/sched/task.h>
12 #include <linux/init.h>
13
14
15
16
17 static void fpu__init_cpu_generic(void)
18 {
19 unsigned long cr0;
20 unsigned long cr4_mask = 0;
21
22 if (boot_cpu_has(X86_FEATURE_FXSR))
23 cr4_mask |= X86_CR4_OSFXSR;
24 if (boot_cpu_has(X86_FEATURE_XMM))
25 cr4_mask |= X86_CR4_OSXMMEXCPT;
26 if (cr4_mask)
27 cr4_set_bits(cr4_mask);
28
29 cr0 = read_cr0();
30 cr0 &= ~(X86_CR0_TS|X86_CR0_EM);
31 if (!boot_cpu_has(X86_FEATURE_FPU))
32 cr0 |= X86_CR0_EM;
33 write_cr0(cr0);
34
35
36 #ifdef CONFIG_MATH_EMULATION
37 if (!boot_cpu_has(X86_FEATURE_FPU))
38 fpstate_init_soft(¤t->thread.fpu.state.soft);
39 else
40 #endif
41 asm volatile ("fninit");
42 }
43
44
45
46
47 void fpu__init_cpu(void)
48 {
49 fpu__init_cpu_generic();
50 fpu__init_cpu_xstate();
51 }
52
53 static bool fpu__probe_without_cpuid(void)
54 {
55 unsigned long cr0;
56 u16 fsw, fcw;
57
58 fsw = fcw = 0xffff;
59
60 cr0 = read_cr0();
61 cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
62 write_cr0(cr0);
63
64 asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw));
65
66 pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw);
67
68 return fsw == 0 && (fcw & 0x103f) == 0x003f;
69 }
70
71 static void fpu__init_system_early_generic(struct cpuinfo_x86 *c)
72 {
73 if (!boot_cpu_has(X86_FEATURE_CPUID) &&
74 !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {
75 if (fpu__probe_without_cpuid())
76 setup_force_cpu_cap(X86_FEATURE_FPU);
77 else
78 setup_clear_cpu_cap(X86_FEATURE_FPU);
79 }
80
81 #ifndef CONFIG_MATH_EMULATION
82 if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
83 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
84 for (;;)
85 asm volatile("hlt");
86 }
87 #endif
88 }
89
90
91
92
93 unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
94 EXPORT_SYMBOL_GPL(mxcsr_feature_mask);
95
96 static void __init fpu__init_system_mxcsr(void)
97 {
98 unsigned int mask = 0;
99
100 if (boot_cpu_has(X86_FEATURE_FXSR)) {
101
102 static struct fxregs_state fxregs __initdata;
103
104 asm volatile("fxsave %0" : "+m" (fxregs));
105
106 mask = fxregs.mxcsr_mask;
107
108
109
110
111
112
113 if (mask == 0)
114 mask = 0x0000ffbf;
115 }
116 mxcsr_feature_mask &= mask;
117 }
118
119
120
121
122 static void __init fpu__init_system_generic(void)
123 {
124
125
126
127
128 fpstate_init(&init_fpstate);
129
130 fpu__init_system_mxcsr();
131 }
132
133
134
135
136
137
138
139 unsigned int fpu_kernel_xstate_size;
140 EXPORT_SYMBOL_GPL(fpu_kernel_xstate_size);
141
142
143 #define TYPE_ALIGN(TYPE) offsetof(struct { char x; TYPE test; }, test)
144
145
146
147
148
149
150
151 #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
152 BUILD_BUG_ON(sizeof(TYPE) != ALIGN(offsetofend(TYPE, MEMBER), \
153 TYPE_ALIGN(TYPE)))
154
155
156
157
158 static void __init fpu__init_task_struct_size(void)
159 {
160 int task_size = sizeof(struct task_struct);
161
162
163
164
165
166 task_size -= sizeof(((struct task_struct *)0)->thread.fpu.state);
167
168
169
170
171
172 task_size += fpu_kernel_xstate_size;
173
174
175
176
177
178
179
180
181 CHECK_MEMBER_AT_END_OF(struct fpu, state);
182 CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
183 CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
184
185 arch_task_struct_size = task_size;
186 }
187
188
189
190
191
192
193
194 static void __init fpu__init_system_xstate_size_legacy(void)
195 {
196 static int on_boot_cpu __initdata = 1;
197
198 WARN_ON_FPU(!on_boot_cpu);
199 on_boot_cpu = 0;
200
201
202
203
204
205
206 if (!boot_cpu_has(X86_FEATURE_FPU)) {
207 fpu_kernel_xstate_size = sizeof(struct swregs_state);
208 } else {
209 if (boot_cpu_has(X86_FEATURE_FXSR))
210 fpu_kernel_xstate_size =
211 sizeof(struct fxregs_state);
212 else
213 fpu_kernel_xstate_size =
214 sizeof(struct fregs_state);
215 }
216
217 fpu_user_xstate_size = fpu_kernel_xstate_size;
218 }
219
220
221
222
223
224
225 u64 __init fpu__get_supported_xfeatures_mask(void)
226 {
227 return XCNTXT_MASK;
228 }
229
230
231 static void __init fpu__init_system_ctx_switch(void)
232 {
233 static bool on_boot_cpu __initdata = 1;
234
235 WARN_ON_FPU(!on_boot_cpu);
236 on_boot_cpu = 0;
237 }
238
239
240
241
242
243 static void __init fpu__init_parse_early_param(void)
244 {
245 char arg[32];
246 char *argptr = arg;
247 int bit;
248
249 #ifdef CONFIG_X86_32
250 if (cmdline_find_option_bool(boot_command_line, "no387"))
251 #ifdef CONFIG_MATH_EMULATION
252 setup_clear_cpu_cap(X86_FEATURE_FPU);
253 #else
254 pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
255 #endif
256
257 if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
258 setup_clear_cpu_cap(X86_FEATURE_FXSR);
259 #endif
260
261 if (cmdline_find_option_bool(boot_command_line, "noxsave"))
262 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
263
264 if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
265 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
266
267 if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
268 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
269
270 if (cmdline_find_option(boot_command_line, "clearcpuid", arg,
271 sizeof(arg)) &&
272 get_option(&argptr, &bit) &&
273 bit >= 0 &&
274 bit < NCAPINTS * 32)
275 setup_clear_cpu_cap(bit);
276 }
277
278
279
280
281
282 void __init fpu__init_system(struct cpuinfo_x86 *c)
283 {
284 fpu__init_parse_early_param();
285 fpu__init_system_early_generic(c);
286
287
288
289
290
291 fpu__init_cpu();
292
293 fpu__init_system_generic();
294 fpu__init_system_xstate_size_legacy();
295 fpu__init_system_xstate();
296 fpu__init_task_struct_size();
297
298 fpu__init_system_ctx_switch();
299 }