1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
4 #define _ASM_X86_NOSPEC_BRANCH_H_
5
6 #include <linux/static_key.h>
7
8 #include <asm/alternative.h>
9 #include <asm/alternative-asm.h>
10 #include <asm/cpufeatures.h>
11 #include <asm/msr-index.h>
12
13 /*
14 * This should be used immediately before a retpoline alternative. It tells
15 * objtool where the retpolines are so that it can make sense of the control
16 * flow by just reading the original instruction(s) and ignoring the
17 * alternatives.
18 */
19 #define ANNOTATE_NOSPEC_ALTERNATIVE \
20 ANNOTATE_IGNORE_ALTERNATIVE
21
22 /*
23 * Fill the CPU return stack buffer.
24 *
25 * Each entry in the RSB, if used for a speculative 'ret', contains an
26 * infinite 'pause; lfence; jmp' loop to capture speculative execution.
27 *
28 * This is required in various cases for retpoline and IBRS-based
29 * mitigations for the Spectre variant 2 vulnerability. Sometimes to
30 * eliminate potentially bogus entries from the RSB, and sometimes
31 * purely to ensure that it doesn't get empty, which on some CPUs would
32 * allow predictions from other (unwanted!) sources to be used.
33 *
34 * We define a CPP macro such that it can be used from both .S files and
35 * inline assembly. It's possible to do a .macro and then include that
36 * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
37 */
38
39 #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */
40 #define RSB_FILL_LOOPS 16 /* To avoid underflow */
41
42 /*
43 * Google experimented with loop-unrolling and this turned out to be
44 * the optimal version — two calls, each with their own speculation
45 * trap should their return address end up getting used, in a loop.
46 */
47 #define __FILL_RETURN_BUFFER(reg, nr, sp) \
48 mov $(nr/2), reg; \
49 771: \
50 call 772f; \
51 773: /* speculation trap */ \
52 pause; \
53 lfence; \
54 jmp 773b; \
55 772: \
56 call 774f; \
57 775: /* speculation trap */ \
58 pause; \
59 lfence; \
60 jmp 775b; \
61 774: \
62 dec reg; \
63 jnz 771b; \
64 add $(BITS_PER_LONG/8) * nr, sp;
65
66 #ifdef __ASSEMBLY__
67
68 /*
69 * This should be used immediately before an indirect jump/call. It tells
70 * objtool the subsequent indirect jump/call is vouched safe for retpoline
71 * builds.
72 */
73 .macro ANNOTATE_RETPOLINE_SAFE
74 .Lannotate_\@:
75 .pushsection .discard.retpoline_safe
76 _ASM_PTR .Lannotate_\@
77 .popsection
78 .endm
79
80 /*
81 * These are the bare retpoline primitives for indirect jmp and call.
82 * Do not use these directly; they only exist to make the ALTERNATIVE
83 * invocation below less ugly.
84 */
85 .macro RETPOLINE_JMP reg:req
86 call .Ldo_rop_\@
87 .Lspec_trap_\@:
88 pause
89 lfence
90 jmp .Lspec_trap_\@
91 .Ldo_rop_\@:
92 mov \reg, (%_ASM_SP)
93 ret
94 .endm
95
96 /*
97 * This is a wrapper around RETPOLINE_JMP so the called function in reg
98 * returns to the instruction after the macro.
99 */
100 .macro RETPOLINE_CALL reg:req
101 jmp .Ldo_call_\@
102 .Ldo_retpoline_jmp_\@:
103 RETPOLINE_JMP \reg
104 .Ldo_call_\@:
105 call .Ldo_retpoline_jmp_\@
106 .endm
107
108 /*
109 * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
110 * indirect jmp/call which may be susceptible to the Spectre variant 2
111 * attack.
112 */
113 .macro JMP_NOSPEC reg:req
114 #ifdef CONFIG_RETPOLINE
115 ANNOTATE_NOSPEC_ALTERNATIVE
116 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg), \
117 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
118 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
119 #else
120 jmp *\reg
121 #endif
122 .endm
123
124 .macro CALL_NOSPEC reg:req
125 #ifdef CONFIG_RETPOLINE
126 ANNOTATE_NOSPEC_ALTERNATIVE
127 ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg), \
128 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
129 __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD
130 #else
131 call *\reg
132 #endif
133 .endm
134
135 /*
136 * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
137 * monstrosity above, manually.
138 */
139 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
140 #ifdef CONFIG_RETPOLINE
141 ANNOTATE_NOSPEC_ALTERNATIVE
142 ALTERNATIVE "jmp .Lskip_rsb_\@", \
143 __stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP)) \
144 \ftr
145 .Lskip_rsb_\@:
146 #endif
147 .endm
148
149 #else /* __ASSEMBLY__ */
150
151 #define ANNOTATE_RETPOLINE_SAFE \
152 "999:\n\t" \
153 ".pushsection .discard.retpoline_safe\n\t" \
154 _ASM_PTR " 999b\n\t" \
155 ".popsection\n\t"
156
157 #ifdef CONFIG_RETPOLINE
158 #ifdef CONFIG_X86_64
159
160 /*
161 * Inline asm uses the %V modifier which is only in newer GCC
162 * which is ensured when CONFIG_RETPOLINE is defined.
163 */
164 # define CALL_NOSPEC \
165 ANNOTATE_NOSPEC_ALTERNATIVE \
166 ALTERNATIVE_2( \
167 ANNOTATE_RETPOLINE_SAFE \
168 "call *%[thunk_target]\n", \
169 "call __x86_indirect_thunk_%V[thunk_target]\n", \
170 X86_FEATURE_RETPOLINE, \
171 "lfence;\n" \
172 ANNOTATE_RETPOLINE_SAFE \
173 "call *%[thunk_target]\n", \
174 X86_FEATURE_RETPOLINE_AMD)
175 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
176
177 #else /* CONFIG_X86_32 */
178 /*
179 * For i386 we use the original ret-equivalent retpoline, because
180 * otherwise we'll run out of registers. We don't care about CET
181 * here, anyway.
182 */
183 # define CALL_NOSPEC \
184 ANNOTATE_NOSPEC_ALTERNATIVE \
185 ALTERNATIVE_2( \
186 ANNOTATE_RETPOLINE_SAFE \
187 "call *%[thunk_target]\n", \
188 " jmp 904f;\n" \
189 " .align 16\n" \
190 "901: call 903f;\n" \
191 "902: pause;\n" \
192 " lfence;\n" \
193 " jmp 902b;\n" \
194 " .align 16\n" \
195 "903: lea 4(%%esp), %%esp;\n" \
196 " pushl %[thunk_target];\n" \
197 " ret;\n" \
198 " .align 16\n" \
199 "904: call 901b;\n", \
200 X86_FEATURE_RETPOLINE, \
201 "lfence;\n" \
202 ANNOTATE_RETPOLINE_SAFE \
203 "call *%[thunk_target]\n", \
204 X86_FEATURE_RETPOLINE_AMD)
205
206 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
207 #endif
208 #else /* No retpoline for C / inline asm */
209 # define CALL_NOSPEC "call *%[thunk_target]\n"
210 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
211 #endif
212
213 /* The Spectre V2 mitigation variants */
214 enum spectre_v2_mitigation {
215 SPECTRE_V2_NONE,
216 SPECTRE_V2_RETPOLINE_GENERIC,
217 SPECTRE_V2_RETPOLINE_AMD,
218 SPECTRE_V2_IBRS_ENHANCED,
219 };
220
221 /* The indirect branch speculation control variants */
222 enum spectre_v2_user_mitigation {
223 SPECTRE_V2_USER_NONE,
224 SPECTRE_V2_USER_STRICT,
225 SPECTRE_V2_USER_STRICT_PREFERRED,
226 SPECTRE_V2_USER_PRCTL,
227 SPECTRE_V2_USER_SECCOMP,
228 };
229
230 /* The Speculative Store Bypass disable variants */
231 enum ssb_mitigation {
232 SPEC_STORE_BYPASS_NONE,
233 SPEC_STORE_BYPASS_DISABLE,
234 SPEC_STORE_BYPASS_PRCTL,
235 SPEC_STORE_BYPASS_SECCOMP,
236 };
237
238 extern char __indirect_thunk_start[];
239 extern char __indirect_thunk_end[];
240
241 /*
242 * On VMEXIT we must ensure that no RSB predictions learned in the guest
243 * can be followed in the host, by overwriting the RSB completely. Both
244 * retpoline and IBRS mitigations for Spectre v2 need this; only on future
245 * CPUs with IBRS_ALL *might* it be avoided.
246 */
247 static inline void vmexit_fill_RSB(void)
248 {
249 #ifdef CONFIG_RETPOLINE
250 unsigned long loops;
251
252 asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
253 ALTERNATIVE("jmp 910f",
254 __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
255 X86_FEATURE_RETPOLINE)
256 "910:"
257 : "=r" (loops), ASM_CALL_CONSTRAINT
258 : : "memory" );
259 #endif
260 }
261
262 static __always_inline
263 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
264 {
265 asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
266 : : "c" (msr),
267 "a" ((u32)val),
268 "d" ((u32)(val >> 32)),
269 [feature] "i" (feature)
270 : "memory");
271 }
272
273 static inline void indirect_branch_prediction_barrier(void)
274 {
275 u64 val = PRED_CMD_IBPB;
276
277 alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB);
278 }
279
280 /* The Intel SPEC CTRL MSR base value cache */
281 extern u64 x86_spec_ctrl_base;
282
283 /*
284 * With retpoline, we must use IBRS to restrict branch prediction
285 * before calling into firmware.
286 *
287 * (Implemented as CPP macros due to header hell.)
288 */
289 #define firmware_restrict_branch_speculation_start() \
290 do { \
291 u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS; \
292 \
293 preempt_disable(); \
294 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \
295 X86_FEATURE_USE_IBRS_FW); \
296 } while (0)
297
298 #define firmware_restrict_branch_speculation_end() \
299 do { \
300 u64 val = x86_spec_ctrl_base; \
301 \
302 alternative_msr_write(MSR_IA32_SPEC_CTRL, val, \
303 X86_FEATURE_USE_IBRS_FW); \
304 preempt_enable(); \
305 } while (0)
306
307 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
308 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
309 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
310
311 DECLARE_STATIC_KEY_FALSE(mds_user_clear);
312 DECLARE_STATIC_KEY_FALSE(mds_idle_clear);
313
314 #include <asm/segment.h>
315
316 /**
317 * mds_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
318 *
319 * This uses the otherwise unused and obsolete VERW instruction in
320 * combination with microcode which triggers a CPU buffer flush when the
321 * instruction is executed.
322 */
323 static inline void mds_clear_cpu_buffers(void)
324 {
325 static const u16 ds = __KERNEL_DS;
326
327 /*
328 * Has to be the memory-operand variant because only that
329 * guarantees the CPU buffer flush functionality according to
330 * documentation. The register-operand variant does not.
331 * Works with any segment selector, but a valid writable
332 * data segment is the fastest variant.
333 *
334 * "cc" clobber is required because VERW modifies ZF.
335 */
336 asm volatile("verw %[ds]" : : [ds] "m" (ds) : "cc");
337 }
338
339 /**
340 * mds_user_clear_cpu_buffers - Mitigation for MDS and TAA vulnerability
341 *
342 * Clear CPU buffers if the corresponding static key is enabled
343 */
344 static inline void mds_user_clear_cpu_buffers(void)
345 {
346 if (static_branch_likely(&mds_user_clear))
347 mds_clear_cpu_buffers();
348 }
349
350 /**
351 * mds_idle_clear_cpu_buffers - Mitigation for MDS vulnerability
352 *
353 * Clear CPU buffers if the corresponding static key is enabled
354 */
355 static inline void mds_idle_clear_cpu_buffers(void)
356 {
357 if (static_branch_likely(&mds_idle_clear))
358 mds_clear_cpu_buffers();
359 }
360
361 #endif /* __ASSEMBLY__ */
362
363 /*
364 * Below is used in the eBPF JIT compiler and emits the byte sequence
365 * for the following assembly:
366 *
367 * With retpolines configured:
368 *
369 * callq do_rop
370 * spec_trap:
371 * pause
372 * lfence
373 * jmp spec_trap
374 * do_rop:
375 * mov %rax,(%rsp) for x86_64
376 * mov %edx,(%esp) for x86_32
377 * retq
378 *
379 * Without retpolines configured:
380 *
381 * jmp *%rax for x86_64
382 * jmp *%edx for x86_32
383 */
384 #ifdef CONFIG_RETPOLINE
385 # ifdef CONFIG_X86_64
386 # define RETPOLINE_RAX_BPF_JIT_SIZE 17
387 # define RETPOLINE_RAX_BPF_JIT() \
388 do { \
389 EMIT1_off32(0xE8, 7); /* callq do_rop */ \
390 /* spec_trap: */ \
391 EMIT2(0xF3, 0x90); /* pause */ \
392 EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
393 EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
394 /* do_rop: */ \
395 EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */ \
396 EMIT1(0xC3); /* retq */ \
397 } while (0)
398 # else /* !CONFIG_X86_64 */
399 # define RETPOLINE_EDX_BPF_JIT() \
400 do { \
401 EMIT1_off32(0xE8, 7); /* call do_rop */ \
402 /* spec_trap: */ \
403 EMIT2(0xF3, 0x90); /* pause */ \
404 EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
405 EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
406 /* do_rop: */ \
407 EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */ \
408 EMIT1(0xC3); /* ret */ \
409 } while (0)
410 # endif
411 #else /* !CONFIG_RETPOLINE */
412 # ifdef CONFIG_X86_64
413 # define RETPOLINE_RAX_BPF_JIT_SIZE 2
414 # define RETPOLINE_RAX_BPF_JIT() \
415 EMIT2(0xFF, 0xE0); /* jmp *%rax */
416 # else /* !CONFIG_X86_64 */
417 # define RETPOLINE_EDX_BPF_JIT() \
418 EMIT2(0xFF, 0xE2) /* jmp *%edx */
419 # endif
420 #endif
421
422 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */