This source file includes following definitions.
- mce_setup
- mce_log
- mce_inject_log
- mce_register_decode_chain
- mce_unregister_decode_chain
- ctl_reg
- status_reg
- addr_reg
- misc_reg
- smca_ctl_reg
- smca_status_reg
- smca_addr_reg
- smca_misc_reg
- __print_mce
- print_mce
- wait_for_panic
- mce_panic
- msr_to_offset
- mce_rdmsrl
- mce_wrmsrl
- mce_gather_info
- mce_available
- mce_schedule_work
- mce_irq_work_cb
- mce_usable_address
- mce_is_memory_error
- whole_page
- mce_is_correctable
- cec_add_mce
- mce_first_notifier
- srao_decode_notifier
- mce_default_notifier
- mce_read_aux
- machine_check_poll
- mce_no_way_out
- mce_timed_out
- mce_reign
- mce_start
- mce_end
- mce_clear_state
- do_memory_failure
- __mc_check_crashing_cpu
- __mc_scan_banks
- do_machine_check
- memory_failure
- mce_adjust_timer_default
- __start_timer
- mce_timer_fn
- mce_timer_kick
- mce_timer_delete_all
- mce_notify_irq
- __mcheck_cpu_mce_banks_init
- __mcheck_cpu_cap_init
- __mcheck_cpu_init_generic
- __mcheck_cpu_init_clear_banks
- __mcheck_cpu_check_banks
- quirk_sandybridge_ifu
- __mcheck_cpu_apply_quirks
- __mcheck_cpu_ancient_init
- __mcheck_cpu_init_early
- mce_centaur_feature_init
- __mcheck_cpu_init_vendor
- __mcheck_cpu_clear_vendor
- mce_start_timer
- __mcheck_cpu_setup_timer
- __mcheck_cpu_init_timer
- filter_mce
- unexpected_machine_check
- do_mce
- mcheck_cpu_init
- mcheck_cpu_clear
- __mce_disable_bank
- mce_disable_bank
- mcheck_enable
- mcheck_init
- mce_disable_error_reporting
- vendor_disable_error_reporting
- mce_syscore_suspend
- mce_syscore_shutdown
- mce_syscore_resume
- mce_cpu_restart
- mce_restart
- mce_disable_cmci
- mce_enable_ce
- attr_to_bank
- show_bank
- set_bank
- set_ignore_ce
- set_cmci_disabled
- store_int_with_restart
- mce_device_release
- mce_device_create
- mce_device_remove
- mce_disable_cpu
- mce_reenable_cpu
- mce_cpu_dead
- mce_cpu_online
- mce_cpu_pre_down
- mce_init_banks
- mcheck_init_device
- mcheck_disable
- mce_get_debugfs_dir
- mce_reset
- fake_panic_get
- fake_panic_set
- mcheck_debugfs_init
- mcheck_debugfs_init
- mcheck_late_init
1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/thread_info.h>
13 #include <linux/capability.h>
14 #include <linux/miscdevice.h>
15 #include <linux/ratelimit.h>
16 #include <linux/rcupdate.h>
17 #include <linux/kobject.h>
18 #include <linux/uaccess.h>
19 #include <linux/kdebug.h>
20 #include <linux/kernel.h>
21 #include <linux/percpu.h>
22 #include <linux/string.h>
23 #include <linux/device.h>
24 #include <linux/syscore_ops.h>
25 #include <linux/delay.h>
26 #include <linux/ctype.h>
27 #include <linux/sched.h>
28 #include <linux/sysfs.h>
29 #include <linux/types.h>
30 #include <linux/slab.h>
31 #include <linux/init.h>
32 #include <linux/kmod.h>
33 #include <linux/poll.h>
34 #include <linux/nmi.h>
35 #include <linux/cpu.h>
36 #include <linux/ras.h>
37 #include <linux/smp.h>
38 #include <linux/fs.h>
39 #include <linux/mm.h>
40 #include <linux/debugfs.h>
41 #include <linux/irq_work.h>
42 #include <linux/export.h>
43 #include <linux/jump_label.h>
44 #include <linux/set_memory.h>
45
46 #include <asm/intel-family.h>
47 #include <asm/processor.h>
48 #include <asm/traps.h>
49 #include <asm/tlbflush.h>
50 #include <asm/mce.h>
51 #include <asm/msr.h>
52 #include <asm/reboot.h>
53
54 #include "internal.h"
55
56 static DEFINE_MUTEX(mce_log_mutex);
57
58
59 static DEFINE_MUTEX(mce_sysfs_mutex);
60
61 #define CREATE_TRACE_POINTS
62 #include <trace/events/mce.h>
63
64 #define SPINUNIT 100
65
66 DEFINE_PER_CPU(unsigned, mce_exception_count);
67
68 DEFINE_PER_CPU_READ_MOSTLY(unsigned int, mce_num_banks);
69
70 struct mce_bank {
71 u64 ctl;
72 bool init;
73 };
74 static DEFINE_PER_CPU_READ_MOSTLY(struct mce_bank[MAX_NR_BANKS], mce_banks_array);
75
76 #define ATTR_LEN 16
77
78 struct mce_bank_dev {
79 struct device_attribute attr;
80 char attrname[ATTR_LEN];
81 u8 bank;
82 };
83 static struct mce_bank_dev mce_bank_devs[MAX_NR_BANKS];
84
85 struct mce_vendor_flags mce_flags __read_mostly;
86
87 struct mca_config mca_cfg __read_mostly = {
88 .bootlog = -1,
89
90
91
92
93
94
95
96 .tolerant = 1,
97 .monarch_timeout = -1
98 };
99
100 static DEFINE_PER_CPU(struct mce, mces_seen);
101 static unsigned long mce_need_notify;
102 static int cpu_missing;
103
104
105
106
107
108 DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
109 [0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
110 };
111
112
113
114
115
116
117
118
119 mce_banks_t mce_banks_ce_disabled;
120
121 static struct work_struct mce_work;
122 static struct irq_work mce_irq_work;
123
124 static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);
125
126
127
128
129
130 BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain);
131
132
133 void mce_setup(struct mce *m)
134 {
135 memset(m, 0, sizeof(struct mce));
136 m->cpu = m->extcpu = smp_processor_id();
137
138 m->time = __ktime_get_real_seconds();
139 m->cpuvendor = boot_cpu_data.x86_vendor;
140 m->cpuid = cpuid_eax(1);
141 m->socketid = cpu_data(m->extcpu).phys_proc_id;
142 m->apicid = cpu_data(m->extcpu).initial_apicid;
143 rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
144
145 if (this_cpu_has(X86_FEATURE_INTEL_PPIN))
146 rdmsrl(MSR_PPIN, m->ppin);
147
148 m->microcode = boot_cpu_data.microcode;
149 }
150
151 DEFINE_PER_CPU(struct mce, injectm);
152 EXPORT_PER_CPU_SYMBOL_GPL(injectm);
153
154 void mce_log(struct mce *m)
155 {
156 if (!mce_gen_pool_add(m))
157 irq_work_queue(&mce_irq_work);
158 }
159
160 void mce_inject_log(struct mce *m)
161 {
162 mutex_lock(&mce_log_mutex);
163 mce_log(m);
164 mutex_unlock(&mce_log_mutex);
165 }
166 EXPORT_SYMBOL_GPL(mce_inject_log);
167
168 static struct notifier_block mce_srao_nb;
169
170
171
172
173
174
175 #define NUM_DEFAULT_NOTIFIERS 3
176 static atomic_t num_notifiers;
177
178 void mce_register_decode_chain(struct notifier_block *nb)
179 {
180 if (WARN_ON(nb->priority > MCE_PRIO_MCELOG && nb->priority < MCE_PRIO_EDAC))
181 return;
182
183 atomic_inc(&num_notifiers);
184
185 blocking_notifier_chain_register(&x86_mce_decoder_chain, nb);
186 }
187 EXPORT_SYMBOL_GPL(mce_register_decode_chain);
188
189 void mce_unregister_decode_chain(struct notifier_block *nb)
190 {
191 atomic_dec(&num_notifiers);
192
193 blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
194 }
195 EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);
196
197 static inline u32 ctl_reg(int bank)
198 {
199 return MSR_IA32_MCx_CTL(bank);
200 }
201
202 static inline u32 status_reg(int bank)
203 {
204 return MSR_IA32_MCx_STATUS(bank);
205 }
206
207 static inline u32 addr_reg(int bank)
208 {
209 return MSR_IA32_MCx_ADDR(bank);
210 }
211
212 static inline u32 misc_reg(int bank)
213 {
214 return MSR_IA32_MCx_MISC(bank);
215 }
216
217 static inline u32 smca_ctl_reg(int bank)
218 {
219 return MSR_AMD64_SMCA_MCx_CTL(bank);
220 }
221
222 static inline u32 smca_status_reg(int bank)
223 {
224 return MSR_AMD64_SMCA_MCx_STATUS(bank);
225 }
226
227 static inline u32 smca_addr_reg(int bank)
228 {
229 return MSR_AMD64_SMCA_MCx_ADDR(bank);
230 }
231
232 static inline u32 smca_misc_reg(int bank)
233 {
234 return MSR_AMD64_SMCA_MCx_MISC(bank);
235 }
236
237 struct mca_msr_regs msr_ops = {
238 .ctl = ctl_reg,
239 .status = status_reg,
240 .addr = addr_reg,
241 .misc = misc_reg
242 };
243
244 static void __print_mce(struct mce *m)
245 {
246 pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n",
247 m->extcpu,
248 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""),
249 m->mcgstatus, m->bank, m->status);
250
251 if (m->ip) {
252 pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
253 !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
254 m->cs, m->ip);
255
256 if (m->cs == __KERNEL_CS)
257 pr_cont("{%pS}", (void *)(unsigned long)m->ip);
258 pr_cont("\n");
259 }
260
261 pr_emerg(HW_ERR "TSC %llx ", m->tsc);
262 if (m->addr)
263 pr_cont("ADDR %llx ", m->addr);
264 if (m->misc)
265 pr_cont("MISC %llx ", m->misc);
266
267 if (mce_flags.smca) {
268 if (m->synd)
269 pr_cont("SYND %llx ", m->synd);
270 if (m->ipid)
271 pr_cont("IPID %llx ", m->ipid);
272 }
273
274 pr_cont("\n");
275
276
277
278
279 pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
280 m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
281 m->microcode);
282 }
283
284 static void print_mce(struct mce *m)
285 {
286 __print_mce(m);
287
288 if (m->cpuvendor != X86_VENDOR_AMD && m->cpuvendor != X86_VENDOR_HYGON)
289 pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
290 }
291
292 #define PANIC_TIMEOUT 5
293
294 static atomic_t mce_panicked;
295
296 static int fake_panic;
297 static atomic_t mce_fake_panicked;
298
299
300 static void wait_for_panic(void)
301 {
302 long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
303
304 preempt_disable();
305 local_irq_enable();
306 while (timeout-- > 0)
307 udelay(1);
308 if (panic_timeout == 0)
309 panic_timeout = mca_cfg.panic_timeout;
310 panic("Panicing machine check CPU died");
311 }
312
313 static void mce_panic(const char *msg, struct mce *final, char *exp)
314 {
315 int apei_err = 0;
316 struct llist_node *pending;
317 struct mce_evt_llist *l;
318
319 if (!fake_panic) {
320
321
322
323 if (atomic_inc_return(&mce_panicked) > 1)
324 wait_for_panic();
325 barrier();
326
327 bust_spinlocks(1);
328 console_verbose();
329 } else {
330
331 if (atomic_inc_return(&mce_fake_panicked) > 1)
332 return;
333 }
334 pending = mce_gen_pool_prepare_records();
335
336 llist_for_each_entry(l, pending, llnode) {
337 struct mce *m = &l->mce;
338 if (!(m->status & MCI_STATUS_UC)) {
339 print_mce(m);
340 if (!apei_err)
341 apei_err = apei_write_mce(m);
342 }
343 }
344
345 llist_for_each_entry(l, pending, llnode) {
346 struct mce *m = &l->mce;
347 if (!(m->status & MCI_STATUS_UC))
348 continue;
349 if (!final || mce_cmp(m, final)) {
350 print_mce(m);
351 if (!apei_err)
352 apei_err = apei_write_mce(m);
353 }
354 }
355 if (final) {
356 print_mce(final);
357 if (!apei_err)
358 apei_err = apei_write_mce(final);
359 }
360 if (cpu_missing)
361 pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n");
362 if (exp)
363 pr_emerg(HW_ERR "Machine check: %s\n", exp);
364 if (!fake_panic) {
365 if (panic_timeout == 0)
366 panic_timeout = mca_cfg.panic_timeout;
367 panic(msg);
368 } else
369 pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
370 }
371
372
373
374 static int msr_to_offset(u32 msr)
375 {
376 unsigned bank = __this_cpu_read(injectm.bank);
377
378 if (msr == mca_cfg.rip_msr)
379 return offsetof(struct mce, ip);
380 if (msr == msr_ops.status(bank))
381 return offsetof(struct mce, status);
382 if (msr == msr_ops.addr(bank))
383 return offsetof(struct mce, addr);
384 if (msr == msr_ops.misc(bank))
385 return offsetof(struct mce, misc);
386 if (msr == MSR_IA32_MCG_STATUS)
387 return offsetof(struct mce, mcgstatus);
388 return -1;
389 }
390
391
392 static u64 mce_rdmsrl(u32 msr)
393 {
394 u64 v;
395
396 if (__this_cpu_read(injectm.finished)) {
397 int offset = msr_to_offset(msr);
398
399 if (offset < 0)
400 return 0;
401 return *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
402 }
403
404 if (rdmsrl_safe(msr, &v)) {
405 WARN_ONCE(1, "mce: Unable to read MSR 0x%x!\n", msr);
406
407
408
409
410
411 v = 0;
412 }
413
414 return v;
415 }
416
417 static void mce_wrmsrl(u32 msr, u64 v)
418 {
419 if (__this_cpu_read(injectm.finished)) {
420 int offset = msr_to_offset(msr);
421
422 if (offset >= 0)
423 *(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
424 return;
425 }
426 wrmsrl(msr, v);
427 }
428
429
430
431
432
433
434 static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)
435 {
436 mce_setup(m);
437
438 m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
439 if (regs) {
440
441
442
443
444 if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
445 m->ip = regs->ip;
446 m->cs = regs->cs;
447
448
449
450
451
452
453 if (v8086_mode(regs))
454 m->cs |= 3;
455 }
456
457 if (mca_cfg.rip_msr)
458 m->ip = mce_rdmsrl(mca_cfg.rip_msr);
459 }
460 }
461
462 int mce_available(struct cpuinfo_x86 *c)
463 {
464 if (mca_cfg.disabled)
465 return 0;
466 return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
467 }
468
469 static void mce_schedule_work(void)
470 {
471 if (!mce_gen_pool_empty())
472 schedule_work(&mce_work);
473 }
474
475 static void mce_irq_work_cb(struct irq_work *entry)
476 {
477 mce_schedule_work();
478 }
479
480
481
482
483
484
485
486 int mce_usable_address(struct mce *m)
487 {
488 if (!(m->status & MCI_STATUS_ADDRV))
489 return 0;
490
491
492 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
493 return 1;
494
495 if (!(m->status & MCI_STATUS_MISCV))
496 return 0;
497
498 if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
499 return 0;
500
501 if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
502 return 0;
503
504 return 1;
505 }
506 EXPORT_SYMBOL_GPL(mce_usable_address);
507
508 bool mce_is_memory_error(struct mce *m)
509 {
510 if (m->cpuvendor == X86_VENDOR_AMD ||
511 m->cpuvendor == X86_VENDOR_HYGON) {
512 return amd_mce_is_memory_error(m);
513 } else if (m->cpuvendor == X86_VENDOR_INTEL) {
514
515
516
517
518
519
520
521
522
523
524
525
526
527 return (m->status & 0xef80) == BIT(7) ||
528 (m->status & 0xef00) == BIT(8) ||
529 (m->status & 0xeffc) == 0xc;
530 }
531
532 return false;
533 }
534 EXPORT_SYMBOL_GPL(mce_is_memory_error);
535
536 static bool whole_page(struct mce *m)
537 {
538 if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV))
539 return true;
540 return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT;
541 }
542
543 bool mce_is_correctable(struct mce *m)
544 {
545 if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED)
546 return false;
547
548 if (m->cpuvendor == X86_VENDOR_HYGON && m->status & MCI_STATUS_DEFERRED)
549 return false;
550
551 if (m->status & MCI_STATUS_UC)
552 return false;
553
554 return true;
555 }
556 EXPORT_SYMBOL_GPL(mce_is_correctable);
557
558 static bool cec_add_mce(struct mce *m)
559 {
560 if (!m)
561 return false;
562
563
564 if (mce_is_memory_error(m) &&
565 mce_is_correctable(m) &&
566 mce_usable_address(m))
567 if (!cec_add_elem(m->addr >> PAGE_SHIFT))
568 return true;
569
570 return false;
571 }
572
573 static int mce_first_notifier(struct notifier_block *nb, unsigned long val,
574 void *data)
575 {
576 struct mce *m = (struct mce *)data;
577
578 if (!m)
579 return NOTIFY_DONE;
580
581 if (cec_add_mce(m))
582 return NOTIFY_STOP;
583
584
585 trace_mce_record(m);
586
587 set_bit(0, &mce_need_notify);
588
589 mce_notify_irq();
590
591 return NOTIFY_DONE;
592 }
593
594 static struct notifier_block first_nb = {
595 .notifier_call = mce_first_notifier,
596 .priority = MCE_PRIO_FIRST,
597 };
598
599 static int srao_decode_notifier(struct notifier_block *nb, unsigned long val,
600 void *data)
601 {
602 struct mce *mce = (struct mce *)data;
603 unsigned long pfn;
604
605 if (!mce)
606 return NOTIFY_DONE;
607
608 if (mce_usable_address(mce) && (mce->severity == MCE_AO_SEVERITY)) {
609 pfn = mce->addr >> PAGE_SHIFT;
610 if (!memory_failure(pfn, 0))
611 set_mce_nospec(pfn, whole_page(mce));
612 }
613
614 return NOTIFY_OK;
615 }
616 static struct notifier_block mce_srao_nb = {
617 .notifier_call = srao_decode_notifier,
618 .priority = MCE_PRIO_SRAO,
619 };
620
621 static int mce_default_notifier(struct notifier_block *nb, unsigned long val,
622 void *data)
623 {
624 struct mce *m = (struct mce *)data;
625
626 if (!m)
627 return NOTIFY_DONE;
628
629 if (atomic_read(&num_notifiers) > NUM_DEFAULT_NOTIFIERS)
630 return NOTIFY_DONE;
631
632 __print_mce(m);
633
634 return NOTIFY_DONE;
635 }
636
637 static struct notifier_block mce_default_nb = {
638 .notifier_call = mce_default_notifier,
639
640 .priority = MCE_PRIO_LOWEST,
641 };
642
643
644
645
646 static void mce_read_aux(struct mce *m, int i)
647 {
648 if (m->status & MCI_STATUS_MISCV)
649 m->misc = mce_rdmsrl(msr_ops.misc(i));
650
651 if (m->status & MCI_STATUS_ADDRV) {
652 m->addr = mce_rdmsrl(msr_ops.addr(i));
653
654
655
656
657 if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
658 u8 shift = MCI_MISC_ADDR_LSB(m->misc);
659 m->addr >>= shift;
660 m->addr <<= shift;
661 }
662
663
664
665
666
667 if (mce_flags.smca) {
668 u8 lsb = (m->addr >> 56) & 0x3f;
669
670 m->addr &= GENMASK_ULL(55, lsb);
671 }
672 }
673
674 if (mce_flags.smca) {
675 m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i));
676
677 if (m->status & MCI_STATUS_SYNDV)
678 m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i));
679 }
680 }
681
682 DEFINE_PER_CPU(unsigned, mce_poll_count);
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699 bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
700 {
701 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
702 bool error_seen = false;
703 struct mce m;
704 int i;
705
706 this_cpu_inc(mce_poll_count);
707
708 mce_gather_info(&m, NULL);
709
710 if (flags & MCP_TIMESTAMP)
711 m.tsc = rdtsc();
712
713 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
714 if (!mce_banks[i].ctl || !test_bit(i, *b))
715 continue;
716
717 m.misc = 0;
718 m.addr = 0;
719 m.bank = i;
720
721 barrier();
722 m.status = mce_rdmsrl(msr_ops.status(i));
723
724
725 if (!(m.status & MCI_STATUS_VAL))
726 continue;
727
728
729
730
731
732 if ((flags & MCP_UC) || !(m.status & MCI_STATUS_UC))
733 goto log_it;
734
735
736
737
738
739
740
741 if (!mca_cfg.ser) {
742 if (m.status & MCI_STATUS_UC)
743 continue;
744 goto log_it;
745 }
746
747
748 if (!(m.status & MCI_STATUS_EN))
749 goto log_it;
750
751
752
753
754
755 if (!(m.status & MCI_STATUS_PCC) && !(m.status & MCI_STATUS_S))
756 goto log_it;
757
758
759
760
761
762
763 continue;
764
765 log_it:
766 error_seen = true;
767
768 mce_read_aux(&m, i);
769
770 m.severity = mce_severity(&m, mca_cfg.tolerant, NULL, false);
771
772
773
774
775
776 if (!(flags & MCP_DONTLOG) && !mca_cfg.dont_log_ce)
777 mce_log(&m);
778 else if (mce_usable_address(&m)) {
779
780
781
782
783
784 if (!mce_gen_pool_add(&m))
785 mce_schedule_work();
786 }
787
788
789
790
791 mce_wrmsrl(msr_ops.status(i), 0);
792 }
793
794
795
796
797
798
799 sync_core();
800
801 return error_seen;
802 }
803 EXPORT_SYMBOL_GPL(machine_check_poll);
804
805
806
807
808
809 static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
810 struct pt_regs *regs)
811 {
812 char *tmp;
813 int i;
814
815 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
816 m->status = mce_rdmsrl(msr_ops.status(i));
817 if (!(m->status & MCI_STATUS_VAL))
818 continue;
819
820 __set_bit(i, validp);
821 if (quirk_no_way_out)
822 quirk_no_way_out(i, m, regs);
823
824 m->bank = i;
825 if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
826 mce_read_aux(m, i);
827 *msg = tmp;
828 return 1;
829 }
830 }
831 return 0;
832 }
833
834
835
836
837
838 static atomic_t mce_executing;
839
840
841
842
843 static atomic_t mce_callin;
844
845
846
847
848 static int mce_timed_out(u64 *t, const char *msg)
849 {
850
851
852
853
854
855
856 rmb();
857 if (atomic_read(&mce_panicked))
858 wait_for_panic();
859 if (!mca_cfg.monarch_timeout)
860 goto out;
861 if ((s64)*t < SPINUNIT) {
862 if (mca_cfg.tolerant <= 1)
863 mce_panic(msg, NULL, NULL);
864 cpu_missing = 1;
865 return 1;
866 }
867 *t -= SPINUNIT;
868 out:
869 touch_nmi_watchdog();
870 return 0;
871 }
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897 static void mce_reign(void)
898 {
899 int cpu;
900 struct mce *m = NULL;
901 int global_worst = 0;
902 char *msg = NULL;
903 char *nmsg = NULL;
904
905
906
907
908
909
910 for_each_possible_cpu(cpu) {
911 int severity = mce_severity(&per_cpu(mces_seen, cpu),
912 mca_cfg.tolerant,
913 &nmsg, true);
914 if (severity > global_worst) {
915 msg = nmsg;
916 global_worst = severity;
917 m = &per_cpu(mces_seen, cpu);
918 }
919 }
920
921
922
923
924
925
926 if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
927 mce_panic("Fatal machine check", m, msg);
928
929
930
931
932
933
934
935
936
937
938
939 if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3)
940 mce_panic("Fatal machine check from unknown source", NULL, NULL);
941
942
943
944
945
946 for_each_possible_cpu(cpu)
947 memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
948 }
949
950 static atomic_t global_nwo;
951
952
953
954
955
956
957
958
959 static int mce_start(int *no_way_out)
960 {
961 int order;
962 int cpus = num_online_cpus();
963 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
964
965 if (!timeout)
966 return -1;
967
968 atomic_add(*no_way_out, &global_nwo);
969
970
971
972
973 order = atomic_inc_return(&mce_callin);
974
975
976
977
978 while (atomic_read(&mce_callin) != cpus) {
979 if (mce_timed_out(&timeout,
980 "Timeout: Not all CPUs entered broadcast exception handler")) {
981 atomic_set(&global_nwo, 0);
982 return -1;
983 }
984 ndelay(SPINUNIT);
985 }
986
987
988
989
990 smp_rmb();
991
992 if (order == 1) {
993
994
995
996 atomic_set(&mce_executing, 1);
997 } else {
998
999
1000
1001
1002
1003
1004 while (atomic_read(&mce_executing) < order) {
1005 if (mce_timed_out(&timeout,
1006 "Timeout: Subject CPUs unable to finish machine check processing")) {
1007 atomic_set(&global_nwo, 0);
1008 return -1;
1009 }
1010 ndelay(SPINUNIT);
1011 }
1012 }
1013
1014
1015
1016
1017 *no_way_out = atomic_read(&global_nwo);
1018
1019 return order;
1020 }
1021
1022
1023
1024
1025
1026 static int mce_end(int order)
1027 {
1028 int ret = -1;
1029 u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
1030
1031 if (!timeout)
1032 goto reset;
1033 if (order < 0)
1034 goto reset;
1035
1036
1037
1038
1039 atomic_inc(&mce_executing);
1040
1041 if (order == 1) {
1042
1043 int cpus = num_online_cpus();
1044
1045
1046
1047
1048
1049 while (atomic_read(&mce_executing) <= cpus) {
1050 if (mce_timed_out(&timeout,
1051 "Timeout: Monarch CPU unable to finish machine check processing"))
1052 goto reset;
1053 ndelay(SPINUNIT);
1054 }
1055
1056 mce_reign();
1057 barrier();
1058 ret = 0;
1059 } else {
1060
1061
1062
1063 while (atomic_read(&mce_executing) != 0) {
1064 if (mce_timed_out(&timeout,
1065 "Timeout: Monarch CPU did not finish machine check processing"))
1066 goto reset;
1067 ndelay(SPINUNIT);
1068 }
1069
1070
1071
1072
1073 return 0;
1074 }
1075
1076
1077
1078
1079 reset:
1080 atomic_set(&global_nwo, 0);
1081 atomic_set(&mce_callin, 0);
1082 barrier();
1083
1084
1085
1086
1087 atomic_set(&mce_executing, 0);
1088 return ret;
1089 }
1090
1091 static void mce_clear_state(unsigned long *toclear)
1092 {
1093 int i;
1094
1095 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1096 if (test_bit(i, toclear))
1097 mce_wrmsrl(msr_ops.status(i), 0);
1098 }
1099 }
1100
1101 static int do_memory_failure(struct mce *m)
1102 {
1103 int flags = MF_ACTION_REQUIRED;
1104 int ret;
1105
1106 pr_err("Uncorrected hardware memory error in user-access at %llx", m->addr);
1107 if (!(m->mcgstatus & MCG_STATUS_RIPV))
1108 flags |= MF_MUST_KILL;
1109 ret = memory_failure(m->addr >> PAGE_SHIFT, flags);
1110 if (ret)
1111 pr_err("Memory error not recovered");
1112 else
1113 set_mce_nospec(m->addr >> PAGE_SHIFT, whole_page(m));
1114 return ret;
1115 }
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130 static bool __mc_check_crashing_cpu(int cpu)
1131 {
1132 if (cpu_is_offline(cpu) ||
1133 (crashing_cpu != -1 && crashing_cpu != cpu)) {
1134 u64 mcgstatus;
1135
1136 mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
1137 if (mcgstatus & MCG_STATUS_RIPV) {
1138 mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1139 return true;
1140 }
1141 }
1142 return false;
1143 }
1144
1145 static void __mc_scan_banks(struct mce *m, struct mce *final,
1146 unsigned long *toclear, unsigned long *valid_banks,
1147 int no_way_out, int *worst)
1148 {
1149 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1150 struct mca_config *cfg = &mca_cfg;
1151 int severity, i;
1152
1153 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1154 __clear_bit(i, toclear);
1155 if (!test_bit(i, valid_banks))
1156 continue;
1157
1158 if (!mce_banks[i].ctl)
1159 continue;
1160
1161 m->misc = 0;
1162 m->addr = 0;
1163 m->bank = i;
1164
1165 m->status = mce_rdmsrl(msr_ops.status(i));
1166 if (!(m->status & MCI_STATUS_VAL))
1167 continue;
1168
1169
1170
1171
1172
1173 if (!(m->status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
1174 !no_way_out)
1175 continue;
1176
1177
1178 add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1179
1180 severity = mce_severity(m, cfg->tolerant, NULL, true);
1181
1182
1183
1184
1185
1186 if ((severity == MCE_KEEP_SEVERITY ||
1187 severity == MCE_UCNA_SEVERITY) && !no_way_out)
1188 continue;
1189
1190 __set_bit(i, toclear);
1191
1192
1193 if (severity == MCE_NO_SEVERITY)
1194 continue;
1195
1196 mce_read_aux(m, i);
1197
1198
1199 m->severity = severity;
1200
1201 mce_log(m);
1202
1203 if (severity > *worst) {
1204 *final = *m;
1205 *worst = severity;
1206 }
1207 }
1208
1209
1210 *m = *final;
1211 }
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225 void do_machine_check(struct pt_regs *regs, long error_code)
1226 {
1227 DECLARE_BITMAP(valid_banks, MAX_NR_BANKS);
1228 DECLARE_BITMAP(toclear, MAX_NR_BANKS);
1229 struct mca_config *cfg = &mca_cfg;
1230 int cpu = smp_processor_id();
1231 char *msg = "Unknown";
1232 struct mce m, *final;
1233 int worst = 0;
1234
1235
1236
1237
1238
1239 int order = -1;
1240
1241
1242
1243
1244
1245 int no_way_out = 0;
1246
1247
1248
1249
1250
1251 int kill_it = 0;
1252
1253
1254
1255
1256
1257 int lmce = 1;
1258
1259 if (__mc_check_crashing_cpu(cpu))
1260 return;
1261
1262 ist_enter(regs);
1263
1264 this_cpu_inc(mce_exception_count);
1265
1266 mce_gather_info(&m, regs);
1267 m.tsc = rdtsc();
1268
1269 final = this_cpu_ptr(&mces_seen);
1270 *final = m;
1271
1272 memset(valid_banks, 0, sizeof(valid_banks));
1273 no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1274
1275 barrier();
1276
1277
1278
1279
1280
1281
1282 if (!(m.mcgstatus & MCG_STATUS_RIPV))
1283 kill_it = 1;
1284
1285
1286
1287
1288
1289 if (m.cpuvendor == X86_VENDOR_INTEL)
1290 lmce = m.mcgstatus & MCG_STATUS_LMCES;
1291
1292
1293
1294
1295
1296
1297
1298
1299 if (lmce) {
1300 if (no_way_out)
1301 mce_panic("Fatal local machine check", &m, msg);
1302 } else {
1303 order = mce_start(&no_way_out);
1304 }
1305
1306 __mc_scan_banks(&m, final, toclear, valid_banks, no_way_out, &worst);
1307
1308 if (!no_way_out)
1309 mce_clear_state(toclear);
1310
1311
1312
1313
1314
1315 if (!lmce) {
1316 if (mce_end(order) < 0)
1317 no_way_out = worst >= MCE_PANIC_SEVERITY;
1318 } else {
1319
1320
1321
1322
1323
1324
1325
1326
1327 if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3) {
1328 mce_severity(&m, cfg->tolerant, &msg, true);
1329 mce_panic("Local fatal machine check!", &m, msg);
1330 }
1331 }
1332
1333
1334
1335
1336
1337 if (cfg->tolerant == 3)
1338 kill_it = 0;
1339 else if (no_way_out)
1340 mce_panic("Fatal machine check on current CPU", &m, msg);
1341
1342 if (worst > 0)
1343 irq_work_queue(&mce_irq_work);
1344
1345 mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1346
1347 sync_core();
1348
1349 if (worst != MCE_AR_SEVERITY && !kill_it)
1350 goto out_ist;
1351
1352
1353 if ((m.cs & 3) == 3) {
1354 ist_begin_non_atomic(regs);
1355 local_irq_enable();
1356
1357 if (kill_it || do_memory_failure(&m))
1358 force_sig(SIGBUS);
1359 local_irq_disable();
1360 ist_end_non_atomic();
1361 } else {
1362 if (!fixup_exception(regs, X86_TRAP_MC, error_code, 0))
1363 mce_panic("Failed kernel mode recovery", &m, NULL);
1364 }
1365
1366 out_ist:
1367 ist_exit(regs);
1368 }
1369 EXPORT_SYMBOL_GPL(do_machine_check);
1370
1371 #ifndef CONFIG_MEMORY_FAILURE
1372 int memory_failure(unsigned long pfn, int flags)
1373 {
1374
1375 BUG_ON(flags & MF_ACTION_REQUIRED);
1376 pr_err("Uncorrected memory error in page 0x%lx ignored\n"
1377 "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
1378 pfn);
1379
1380 return 0;
1381 }
1382 #endif
1383
1384
1385
1386
1387
1388
1389 static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
1390
1391 static DEFINE_PER_CPU(unsigned long, mce_next_interval);
1392 static DEFINE_PER_CPU(struct timer_list, mce_timer);
1393
1394 static unsigned long mce_adjust_timer_default(unsigned long interval)
1395 {
1396 return interval;
1397 }
1398
1399 static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
1400
1401 static void __start_timer(struct timer_list *t, unsigned long interval)
1402 {
1403 unsigned long when = jiffies + interval;
1404 unsigned long flags;
1405
1406 local_irq_save(flags);
1407
1408 if (!timer_pending(t) || time_before(when, t->expires))
1409 mod_timer(t, round_jiffies(when));
1410
1411 local_irq_restore(flags);
1412 }
1413
1414 static void mce_timer_fn(struct timer_list *t)
1415 {
1416 struct timer_list *cpu_t = this_cpu_ptr(&mce_timer);
1417 unsigned long iv;
1418
1419 WARN_ON(cpu_t != t);
1420
1421 iv = __this_cpu_read(mce_next_interval);
1422
1423 if (mce_available(this_cpu_ptr(&cpu_info))) {
1424 machine_check_poll(0, this_cpu_ptr(&mce_poll_banks));
1425
1426 if (mce_intel_cmci_poll()) {
1427 iv = mce_adjust_timer(iv);
1428 goto done;
1429 }
1430 }
1431
1432
1433
1434
1435
1436 if (mce_notify_irq())
1437 iv = max(iv / 2, (unsigned long) HZ/100);
1438 else
1439 iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1440
1441 done:
1442 __this_cpu_write(mce_next_interval, iv);
1443 __start_timer(t, iv);
1444 }
1445
1446
1447
1448
1449 void mce_timer_kick(unsigned long interval)
1450 {
1451 struct timer_list *t = this_cpu_ptr(&mce_timer);
1452 unsigned long iv = __this_cpu_read(mce_next_interval);
1453
1454 __start_timer(t, interval);
1455
1456 if (interval < iv)
1457 __this_cpu_write(mce_next_interval, interval);
1458 }
1459
1460
1461 static void mce_timer_delete_all(void)
1462 {
1463 int cpu;
1464
1465 for_each_online_cpu(cpu)
1466 del_timer_sync(&per_cpu(mce_timer, cpu));
1467 }
1468
1469
1470
1471
1472
1473
1474 int mce_notify_irq(void)
1475 {
1476
1477 static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);
1478
1479 if (test_and_clear_bit(0, &mce_need_notify)) {
1480 mce_work_trigger();
1481
1482 if (__ratelimit(&ratelimit))
1483 pr_info(HW_ERR "Machine check events logged\n");
1484
1485 return 1;
1486 }
1487 return 0;
1488 }
1489 EXPORT_SYMBOL_GPL(mce_notify_irq);
1490
1491 static void __mcheck_cpu_mce_banks_init(void)
1492 {
1493 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1494 u8 n_banks = this_cpu_read(mce_num_banks);
1495 int i;
1496
1497 for (i = 0; i < n_banks; i++) {
1498 struct mce_bank *b = &mce_banks[i];
1499
1500
1501
1502
1503
1504
1505 b->ctl = -1ULL;
1506 b->init = 1;
1507 }
1508 }
1509
1510
1511
1512
1513 static void __mcheck_cpu_cap_init(void)
1514 {
1515 u64 cap;
1516 u8 b;
1517
1518 rdmsrl(MSR_IA32_MCG_CAP, cap);
1519
1520 b = cap & MCG_BANKCNT_MASK;
1521
1522 if (b > MAX_NR_BANKS) {
1523 pr_warn("CPU%d: Using only %u machine check banks out of %u\n",
1524 smp_processor_id(), MAX_NR_BANKS, b);
1525 b = MAX_NR_BANKS;
1526 }
1527
1528 this_cpu_write(mce_num_banks, b);
1529
1530 __mcheck_cpu_mce_banks_init();
1531
1532
1533 if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1534 mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
1535
1536 if (cap & MCG_SER_P)
1537 mca_cfg.ser = 1;
1538 }
1539
1540 static void __mcheck_cpu_init_generic(void)
1541 {
1542 enum mcp_flags m_fl = 0;
1543 mce_banks_t all_banks;
1544 u64 cap;
1545
1546 if (!mca_cfg.bootlog)
1547 m_fl = MCP_DONTLOG;
1548
1549
1550
1551
1552 bitmap_fill(all_banks, MAX_NR_BANKS);
1553 machine_check_poll(MCP_UC | m_fl, &all_banks);
1554
1555 cr4_set_bits(X86_CR4_MCE);
1556
1557 rdmsrl(MSR_IA32_MCG_CAP, cap);
1558 if (cap & MCG_CTL_P)
1559 wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
1560 }
1561
1562 static void __mcheck_cpu_init_clear_banks(void)
1563 {
1564 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1565 int i;
1566
1567 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1568 struct mce_bank *b = &mce_banks[i];
1569
1570 if (!b->init)
1571 continue;
1572 wrmsrl(msr_ops.ctl(i), b->ctl);
1573 wrmsrl(msr_ops.status(i), 0);
1574 }
1575 }
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587 static void __mcheck_cpu_check_banks(void)
1588 {
1589 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1590 u64 msrval;
1591 int i;
1592
1593 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
1594 struct mce_bank *b = &mce_banks[i];
1595
1596 if (!b->init)
1597 continue;
1598
1599 rdmsrl(msr_ops.ctl(i), msrval);
1600 b->init = !!msrval;
1601 }
1602 }
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612 static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
1613 {
1614 if (bank != 0)
1615 return;
1616 if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
1617 return;
1618 if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
1619 MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
1620 MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
1621 MCACOD)) !=
1622 (MCI_STATUS_UC|MCI_STATUS_EN|
1623 MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
1624 MCI_STATUS_AR|MCACOD_INSTR))
1625 return;
1626
1627 m->mcgstatus |= MCG_STATUS_EIPV;
1628 m->ip = regs->ip;
1629 m->cs = regs->cs;
1630 }
1631
1632
1633 static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1634 {
1635 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
1636 struct mca_config *cfg = &mca_cfg;
1637
1638 if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1639 pr_info("unknown CPU type - not enabling MCE support\n");
1640 return -EOPNOTSUPP;
1641 }
1642
1643
1644 if (c->x86_vendor == X86_VENDOR_AMD) {
1645 if (c->x86 == 15 && this_cpu_read(mce_num_banks) > 4) {
1646
1647
1648
1649
1650
1651 clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
1652 }
1653 if (c->x86 < 0x11 && cfg->bootlog < 0) {
1654
1655
1656
1657
1658 cfg->bootlog = 0;
1659 }
1660
1661
1662
1663
1664 if (c->x86 == 6 && this_cpu_read(mce_num_banks) > 0)
1665 mce_banks[0].ctl = 0;
1666
1667
1668
1669
1670
1671 if (c->x86 == 0x15 && c->x86_model <= 0xf)
1672 mce_flags.overflow_recov = 1;
1673
1674 }
1675
1676 if (c->x86_vendor == X86_VENDOR_INTEL) {
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686 if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0)
1687 mce_banks[0].init = 0;
1688
1689
1690
1691
1692
1693 if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1694 cfg->monarch_timeout < 0)
1695 cfg->monarch_timeout = USEC_PER_SEC;
1696
1697
1698
1699
1700
1701 if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
1702 cfg->bootlog = 0;
1703
1704 if (c->x86 == 6 && c->x86_model == 45)
1705 quirk_no_way_out = quirk_sandybridge_ifu;
1706 }
1707 if (cfg->monarch_timeout < 0)
1708 cfg->monarch_timeout = 0;
1709 if (cfg->bootlog != 0)
1710 cfg->panic_timeout = 30;
1711
1712 return 0;
1713 }
1714
1715 static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1716 {
1717 if (c->x86 != 5)
1718 return 0;
1719
1720 switch (c->x86_vendor) {
1721 case X86_VENDOR_INTEL:
1722 intel_p5_mcheck_init(c);
1723 return 1;
1724 break;
1725 case X86_VENDOR_CENTAUR:
1726 winchip_mcheck_init(c);
1727 return 1;
1728 break;
1729 default:
1730 return 0;
1731 }
1732
1733 return 0;
1734 }
1735
1736
1737
1738
1739 static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c)
1740 {
1741 if (c->x86_vendor == X86_VENDOR_AMD || c->x86_vendor == X86_VENDOR_HYGON) {
1742 mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV);
1743 mce_flags.succor = !!cpu_has(c, X86_FEATURE_SUCCOR);
1744 mce_flags.smca = !!cpu_has(c, X86_FEATURE_SMCA);
1745
1746 if (mce_flags.smca) {
1747 msr_ops.ctl = smca_ctl_reg;
1748 msr_ops.status = smca_status_reg;
1749 msr_ops.addr = smca_addr_reg;
1750 msr_ops.misc = smca_misc_reg;
1751 }
1752 }
1753 }
1754
1755 static void mce_centaur_feature_init(struct cpuinfo_x86 *c)
1756 {
1757 struct mca_config *cfg = &mca_cfg;
1758
1759
1760
1761
1762
1763 if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) ||
1764 c->x86 > 6) {
1765 if (cfg->monarch_timeout < 0)
1766 cfg->monarch_timeout = USEC_PER_SEC;
1767 }
1768 }
1769
1770 static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
1771 {
1772 switch (c->x86_vendor) {
1773 case X86_VENDOR_INTEL:
1774 mce_intel_feature_init(c);
1775 mce_adjust_timer = cmci_intel_adjust_timer;
1776 break;
1777
1778 case X86_VENDOR_AMD: {
1779 mce_amd_feature_init(c);
1780 break;
1781 }
1782
1783 case X86_VENDOR_HYGON:
1784 mce_hygon_feature_init(c);
1785 break;
1786
1787 case X86_VENDOR_CENTAUR:
1788 mce_centaur_feature_init(c);
1789 break;
1790
1791 default:
1792 break;
1793 }
1794 }
1795
1796 static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
1797 {
1798 switch (c->x86_vendor) {
1799 case X86_VENDOR_INTEL:
1800 mce_intel_feature_clear(c);
1801 break;
1802 default:
1803 break;
1804 }
1805 }
1806
1807 static void mce_start_timer(struct timer_list *t)
1808 {
1809 unsigned long iv = check_interval * HZ;
1810
1811 if (mca_cfg.ignore_ce || !iv)
1812 return;
1813
1814 this_cpu_write(mce_next_interval, iv);
1815 __start_timer(t, iv);
1816 }
1817
1818 static void __mcheck_cpu_setup_timer(void)
1819 {
1820 struct timer_list *t = this_cpu_ptr(&mce_timer);
1821
1822 timer_setup(t, mce_timer_fn, TIMER_PINNED);
1823 }
1824
1825 static void __mcheck_cpu_init_timer(void)
1826 {
1827 struct timer_list *t = this_cpu_ptr(&mce_timer);
1828
1829 timer_setup(t, mce_timer_fn, TIMER_PINNED);
1830 mce_start_timer(t);
1831 }
1832
1833 bool filter_mce(struct mce *m)
1834 {
1835 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1836 return amd_filter_mce(m);
1837
1838 return false;
1839 }
1840
1841
1842 static void unexpected_machine_check(struct pt_regs *regs, long error_code)
1843 {
1844 pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
1845 smp_processor_id());
1846 }
1847
1848
1849 void (*machine_check_vector)(struct pt_regs *, long error_code) =
1850 unexpected_machine_check;
1851
1852 dotraplinkage void do_mce(struct pt_regs *regs, long error_code)
1853 {
1854 machine_check_vector(regs, error_code);
1855 }
1856
1857
1858
1859
1860
1861 void mcheck_cpu_init(struct cpuinfo_x86 *c)
1862 {
1863 if (mca_cfg.disabled)
1864 return;
1865
1866 if (__mcheck_cpu_ancient_init(c))
1867 return;
1868
1869 if (!mce_available(c))
1870 return;
1871
1872 __mcheck_cpu_cap_init();
1873
1874 if (__mcheck_cpu_apply_quirks(c) < 0) {
1875 mca_cfg.disabled = 1;
1876 return;
1877 }
1878
1879 if (mce_gen_pool_init()) {
1880 mca_cfg.disabled = 1;
1881 pr_emerg("Couldn't allocate MCE records pool!\n");
1882 return;
1883 }
1884
1885 machine_check_vector = do_machine_check;
1886
1887 __mcheck_cpu_init_early(c);
1888 __mcheck_cpu_init_generic();
1889 __mcheck_cpu_init_vendor(c);
1890 __mcheck_cpu_init_clear_banks();
1891 __mcheck_cpu_check_banks();
1892 __mcheck_cpu_setup_timer();
1893 }
1894
1895
1896
1897
1898 void mcheck_cpu_clear(struct cpuinfo_x86 *c)
1899 {
1900 if (mca_cfg.disabled)
1901 return;
1902
1903 if (!mce_available(c))
1904 return;
1905
1906
1907
1908
1909
1910 __mcheck_cpu_clear_vendor(c);
1911
1912 }
1913
1914 static void __mce_disable_bank(void *arg)
1915 {
1916 int bank = *((int *)arg);
1917 __clear_bit(bank, this_cpu_ptr(mce_poll_banks));
1918 cmci_disable_bank(bank);
1919 }
1920
1921 void mce_disable_bank(int bank)
1922 {
1923 if (bank >= this_cpu_read(mce_num_banks)) {
1924 pr_warn(FW_BUG
1925 "Ignoring request to disable invalid MCA bank %d.\n",
1926 bank);
1927 return;
1928 }
1929 set_bit(bank, mce_banks_ce_disabled);
1930 on_each_cpu(__mce_disable_bank, &bank, 1);
1931 }
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948 static int __init mcheck_enable(char *str)
1949 {
1950 struct mca_config *cfg = &mca_cfg;
1951
1952 if (*str == 0) {
1953 enable_p5_mce();
1954 return 1;
1955 }
1956 if (*str == '=')
1957 str++;
1958 if (!strcmp(str, "off"))
1959 cfg->disabled = 1;
1960 else if (!strcmp(str, "no_cmci"))
1961 cfg->cmci_disabled = true;
1962 else if (!strcmp(str, "no_lmce"))
1963 cfg->lmce_disabled = 1;
1964 else if (!strcmp(str, "dont_log_ce"))
1965 cfg->dont_log_ce = true;
1966 else if (!strcmp(str, "ignore_ce"))
1967 cfg->ignore_ce = true;
1968 else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
1969 cfg->bootlog = (str[0] == 'b');
1970 else if (!strcmp(str, "bios_cmci_threshold"))
1971 cfg->bios_cmci_threshold = 1;
1972 else if (!strcmp(str, "recovery"))
1973 cfg->recovery = 1;
1974 else if (isdigit(str[0])) {
1975 if (get_option(&str, &cfg->tolerant) == 2)
1976 get_option(&str, &(cfg->monarch_timeout));
1977 } else {
1978 pr_info("mce argument %s ignored. Please use /sys\n", str);
1979 return 0;
1980 }
1981 return 1;
1982 }
1983 __setup("mce", mcheck_enable);
1984
1985 int __init mcheck_init(void)
1986 {
1987 mcheck_intel_therm_init();
1988 mce_register_decode_chain(&first_nb);
1989 mce_register_decode_chain(&mce_srao_nb);
1990 mce_register_decode_chain(&mce_default_nb);
1991 mcheck_vendor_init_severity();
1992
1993 INIT_WORK(&mce_work, mce_gen_pool_process);
1994 init_irq_work(&mce_irq_work, mce_irq_work_cb);
1995
1996 return 0;
1997 }
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007 static void mce_disable_error_reporting(void)
2008 {
2009 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2010 int i;
2011
2012 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2013 struct mce_bank *b = &mce_banks[i];
2014
2015 if (b->init)
2016 wrmsrl(msr_ops.ctl(i), 0);
2017 }
2018 return;
2019 }
2020
2021 static void vendor_disable_error_reporting(void)
2022 {
2023
2024
2025
2026
2027
2028
2029
2030 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL ||
2031 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ||
2032 boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
2033 return;
2034
2035 mce_disable_error_reporting();
2036 }
2037
2038 static int mce_syscore_suspend(void)
2039 {
2040 vendor_disable_error_reporting();
2041 return 0;
2042 }
2043
2044 static void mce_syscore_shutdown(void)
2045 {
2046 vendor_disable_error_reporting();
2047 }
2048
2049
2050
2051
2052
2053
2054 static void mce_syscore_resume(void)
2055 {
2056 __mcheck_cpu_init_generic();
2057 __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
2058 __mcheck_cpu_init_clear_banks();
2059 }
2060
2061 static struct syscore_ops mce_syscore_ops = {
2062 .suspend = mce_syscore_suspend,
2063 .shutdown = mce_syscore_shutdown,
2064 .resume = mce_syscore_resume,
2065 };
2066
2067
2068
2069
2070
2071 static void mce_cpu_restart(void *data)
2072 {
2073 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2074 return;
2075 __mcheck_cpu_init_generic();
2076 __mcheck_cpu_init_clear_banks();
2077 __mcheck_cpu_init_timer();
2078 }
2079
2080
2081 static void mce_restart(void)
2082 {
2083 mce_timer_delete_all();
2084 on_each_cpu(mce_cpu_restart, NULL, 1);
2085 }
2086
2087
2088 static void mce_disable_cmci(void *data)
2089 {
2090 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2091 return;
2092 cmci_clear();
2093 }
2094
2095 static void mce_enable_ce(void *all)
2096 {
2097 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2098 return;
2099 cmci_reenable();
2100 cmci_recheck();
2101 if (all)
2102 __mcheck_cpu_init_timer();
2103 }
2104
2105 static struct bus_type mce_subsys = {
2106 .name = "machinecheck",
2107 .dev_name = "machinecheck",
2108 };
2109
2110 DEFINE_PER_CPU(struct device *, mce_device);
2111
2112 static inline struct mce_bank_dev *attr_to_bank(struct device_attribute *attr)
2113 {
2114 return container_of(attr, struct mce_bank_dev, attr);
2115 }
2116
2117 static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2118 char *buf)
2119 {
2120 u8 bank = attr_to_bank(attr)->bank;
2121 struct mce_bank *b;
2122
2123 if (bank >= per_cpu(mce_num_banks, s->id))
2124 return -EINVAL;
2125
2126 b = &per_cpu(mce_banks_array, s->id)[bank];
2127
2128 if (!b->init)
2129 return -ENODEV;
2130
2131 return sprintf(buf, "%llx\n", b->ctl);
2132 }
2133
2134 static ssize_t set_bank(struct device *s, struct device_attribute *attr,
2135 const char *buf, size_t size)
2136 {
2137 u8 bank = attr_to_bank(attr)->bank;
2138 struct mce_bank *b;
2139 u64 new;
2140
2141 if (kstrtou64(buf, 0, &new) < 0)
2142 return -EINVAL;
2143
2144 if (bank >= per_cpu(mce_num_banks, s->id))
2145 return -EINVAL;
2146
2147 b = &per_cpu(mce_banks_array, s->id)[bank];
2148
2149 if (!b->init)
2150 return -ENODEV;
2151
2152 b->ctl = new;
2153 mce_restart();
2154
2155 return size;
2156 }
2157
2158 static ssize_t set_ignore_ce(struct device *s,
2159 struct device_attribute *attr,
2160 const char *buf, size_t size)
2161 {
2162 u64 new;
2163
2164 if (kstrtou64(buf, 0, &new) < 0)
2165 return -EINVAL;
2166
2167 mutex_lock(&mce_sysfs_mutex);
2168 if (mca_cfg.ignore_ce ^ !!new) {
2169 if (new) {
2170
2171 mce_timer_delete_all();
2172 on_each_cpu(mce_disable_cmci, NULL, 1);
2173 mca_cfg.ignore_ce = true;
2174 } else {
2175
2176 mca_cfg.ignore_ce = false;
2177 on_each_cpu(mce_enable_ce, (void *)1, 1);
2178 }
2179 }
2180 mutex_unlock(&mce_sysfs_mutex);
2181
2182 return size;
2183 }
2184
2185 static ssize_t set_cmci_disabled(struct device *s,
2186 struct device_attribute *attr,
2187 const char *buf, size_t size)
2188 {
2189 u64 new;
2190
2191 if (kstrtou64(buf, 0, &new) < 0)
2192 return -EINVAL;
2193
2194 mutex_lock(&mce_sysfs_mutex);
2195 if (mca_cfg.cmci_disabled ^ !!new) {
2196 if (new) {
2197
2198 on_each_cpu(mce_disable_cmci, NULL, 1);
2199 mca_cfg.cmci_disabled = true;
2200 } else {
2201
2202 mca_cfg.cmci_disabled = false;
2203 on_each_cpu(mce_enable_ce, NULL, 1);
2204 }
2205 }
2206 mutex_unlock(&mce_sysfs_mutex);
2207
2208 return size;
2209 }
2210
2211 static ssize_t store_int_with_restart(struct device *s,
2212 struct device_attribute *attr,
2213 const char *buf, size_t size)
2214 {
2215 unsigned long old_check_interval = check_interval;
2216 ssize_t ret = device_store_ulong(s, attr, buf, size);
2217
2218 if (check_interval == old_check_interval)
2219 return ret;
2220
2221 mutex_lock(&mce_sysfs_mutex);
2222 mce_restart();
2223 mutex_unlock(&mce_sysfs_mutex);
2224
2225 return ret;
2226 }
2227
2228 static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant);
2229 static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2230 static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
2231
2232 static struct dev_ext_attribute dev_attr_check_interval = {
2233 __ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2234 &check_interval
2235 };
2236
2237 static struct dev_ext_attribute dev_attr_ignore_ce = {
2238 __ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
2239 &mca_cfg.ignore_ce
2240 };
2241
2242 static struct dev_ext_attribute dev_attr_cmci_disabled = {
2243 __ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
2244 &mca_cfg.cmci_disabled
2245 };
2246
2247 static struct device_attribute *mce_device_attrs[] = {
2248 &dev_attr_tolerant.attr,
2249 &dev_attr_check_interval.attr,
2250 #ifdef CONFIG_X86_MCELOG_LEGACY
2251 &dev_attr_trigger,
2252 #endif
2253 &dev_attr_monarch_timeout.attr,
2254 &dev_attr_dont_log_ce.attr,
2255 &dev_attr_ignore_ce.attr,
2256 &dev_attr_cmci_disabled.attr,
2257 NULL
2258 };
2259
2260 static cpumask_var_t mce_device_initialized;
2261
2262 static void mce_device_release(struct device *dev)
2263 {
2264 kfree(dev);
2265 }
2266
2267
2268 static int mce_device_create(unsigned int cpu)
2269 {
2270 struct device *dev;
2271 int err;
2272 int i, j;
2273
2274 if (!mce_available(&boot_cpu_data))
2275 return -EIO;
2276
2277 dev = per_cpu(mce_device, cpu);
2278 if (dev)
2279 return 0;
2280
2281 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2282 if (!dev)
2283 return -ENOMEM;
2284 dev->id = cpu;
2285 dev->bus = &mce_subsys;
2286 dev->release = &mce_device_release;
2287
2288 err = device_register(dev);
2289 if (err) {
2290 put_device(dev);
2291 return err;
2292 }
2293
2294 for (i = 0; mce_device_attrs[i]; i++) {
2295 err = device_create_file(dev, mce_device_attrs[i]);
2296 if (err)
2297 goto error;
2298 }
2299 for (j = 0; j < per_cpu(mce_num_banks, cpu); j++) {
2300 err = device_create_file(dev, &mce_bank_devs[j].attr);
2301 if (err)
2302 goto error2;
2303 }
2304 cpumask_set_cpu(cpu, mce_device_initialized);
2305 per_cpu(mce_device, cpu) = dev;
2306
2307 return 0;
2308 error2:
2309 while (--j >= 0)
2310 device_remove_file(dev, &mce_bank_devs[j].attr);
2311 error:
2312 while (--i >= 0)
2313 device_remove_file(dev, mce_device_attrs[i]);
2314
2315 device_unregister(dev);
2316
2317 return err;
2318 }
2319
2320 static void mce_device_remove(unsigned int cpu)
2321 {
2322 struct device *dev = per_cpu(mce_device, cpu);
2323 int i;
2324
2325 if (!cpumask_test_cpu(cpu, mce_device_initialized))
2326 return;
2327
2328 for (i = 0; mce_device_attrs[i]; i++)
2329 device_remove_file(dev, mce_device_attrs[i]);
2330
2331 for (i = 0; i < per_cpu(mce_num_banks, cpu); i++)
2332 device_remove_file(dev, &mce_bank_devs[i].attr);
2333
2334 device_unregister(dev);
2335 cpumask_clear_cpu(cpu, mce_device_initialized);
2336 per_cpu(mce_device, cpu) = NULL;
2337 }
2338
2339
2340 static void mce_disable_cpu(void)
2341 {
2342 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2343 return;
2344
2345 if (!cpuhp_tasks_frozen)
2346 cmci_clear();
2347
2348 vendor_disable_error_reporting();
2349 }
2350
2351 static void mce_reenable_cpu(void)
2352 {
2353 struct mce_bank *mce_banks = this_cpu_ptr(mce_banks_array);
2354 int i;
2355
2356 if (!mce_available(raw_cpu_ptr(&cpu_info)))
2357 return;
2358
2359 if (!cpuhp_tasks_frozen)
2360 cmci_reenable();
2361 for (i = 0; i < this_cpu_read(mce_num_banks); i++) {
2362 struct mce_bank *b = &mce_banks[i];
2363
2364 if (b->init)
2365 wrmsrl(msr_ops.ctl(i), b->ctl);
2366 }
2367 }
2368
2369 static int mce_cpu_dead(unsigned int cpu)
2370 {
2371 mce_intel_hcpu_update(cpu);
2372
2373
2374 if (!cpuhp_tasks_frozen)
2375 cmci_rediscover();
2376 return 0;
2377 }
2378
2379 static int mce_cpu_online(unsigned int cpu)
2380 {
2381 struct timer_list *t = this_cpu_ptr(&mce_timer);
2382 int ret;
2383
2384 mce_device_create(cpu);
2385
2386 ret = mce_threshold_create_device(cpu);
2387 if (ret) {
2388 mce_device_remove(cpu);
2389 return ret;
2390 }
2391 mce_reenable_cpu();
2392 mce_start_timer(t);
2393 return 0;
2394 }
2395
2396 static int mce_cpu_pre_down(unsigned int cpu)
2397 {
2398 struct timer_list *t = this_cpu_ptr(&mce_timer);
2399
2400 mce_disable_cpu();
2401 del_timer_sync(t);
2402 mce_threshold_remove_device(cpu);
2403 mce_device_remove(cpu);
2404 return 0;
2405 }
2406
2407 static __init void mce_init_banks(void)
2408 {
2409 int i;
2410
2411 for (i = 0; i < MAX_NR_BANKS; i++) {
2412 struct mce_bank_dev *b = &mce_bank_devs[i];
2413 struct device_attribute *a = &b->attr;
2414
2415 b->bank = i;
2416
2417 sysfs_attr_init(&a->attr);
2418 a->attr.name = b->attrname;
2419 snprintf(b->attrname, ATTR_LEN, "bank%d", i);
2420
2421 a->attr.mode = 0644;
2422 a->show = show_bank;
2423 a->store = set_bank;
2424 }
2425 }
2426
2427 static __init int mcheck_init_device(void)
2428 {
2429 int err;
2430
2431
2432
2433
2434
2435 MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63);
2436
2437 if (!mce_available(&boot_cpu_data)) {
2438 err = -EIO;
2439 goto err_out;
2440 }
2441
2442 if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
2443 err = -ENOMEM;
2444 goto err_out;
2445 }
2446
2447 mce_init_banks();
2448
2449 err = subsys_system_register(&mce_subsys, NULL);
2450 if (err)
2451 goto err_out_mem;
2452
2453 err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
2454 mce_cpu_dead);
2455 if (err)
2456 goto err_out_mem;
2457
2458 err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
2459 mce_cpu_online, mce_cpu_pre_down);
2460 if (err < 0)
2461 goto err_out_online;
2462
2463 register_syscore_ops(&mce_syscore_ops);
2464
2465 return 0;
2466
2467 err_out_online:
2468 cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
2469
2470 err_out_mem:
2471 free_cpumask_var(mce_device_initialized);
2472
2473 err_out:
2474 pr_err("Unable to init MCE device (rc: %d)\n", err);
2475
2476 return err;
2477 }
2478 device_initcall_sync(mcheck_init_device);
2479
2480
2481
2482
2483 static int __init mcheck_disable(char *str)
2484 {
2485 mca_cfg.disabled = 1;
2486 return 1;
2487 }
2488 __setup("nomce", mcheck_disable);
2489
2490 #ifdef CONFIG_DEBUG_FS
2491 struct dentry *mce_get_debugfs_dir(void)
2492 {
2493 static struct dentry *dmce;
2494
2495 if (!dmce)
2496 dmce = debugfs_create_dir("mce", NULL);
2497
2498 return dmce;
2499 }
2500
2501 static void mce_reset(void)
2502 {
2503 cpu_missing = 0;
2504 atomic_set(&mce_fake_panicked, 0);
2505 atomic_set(&mce_executing, 0);
2506 atomic_set(&mce_callin, 0);
2507 atomic_set(&global_nwo, 0);
2508 }
2509
2510 static int fake_panic_get(void *data, u64 *val)
2511 {
2512 *val = fake_panic;
2513 return 0;
2514 }
2515
2516 static int fake_panic_set(void *data, u64 val)
2517 {
2518 mce_reset();
2519 fake_panic = val;
2520 return 0;
2521 }
2522
2523 DEFINE_DEBUGFS_ATTRIBUTE(fake_panic_fops, fake_panic_get, fake_panic_set,
2524 "%llu\n");
2525
2526 static void __init mcheck_debugfs_init(void)
2527 {
2528 struct dentry *dmce;
2529
2530 dmce = mce_get_debugfs_dir();
2531 debugfs_create_file_unsafe("fake_panic", 0444, dmce, NULL,
2532 &fake_panic_fops);
2533 }
2534 #else
2535 static void __init mcheck_debugfs_init(void) { }
2536 #endif
2537
2538 DEFINE_STATIC_KEY_FALSE(mcsafe_key);
2539 EXPORT_SYMBOL_GPL(mcsafe_key);
2540
2541 static int __init mcheck_late_init(void)
2542 {
2543 if (mca_cfg.recovery)
2544 static_branch_inc(&mcsafe_key);
2545
2546 mcheck_debugfs_init();
2547 cec_init();
2548
2549
2550
2551
2552
2553 mce_schedule_work();
2554
2555 return 0;
2556 }
2557 late_initcall(mcheck_late_init);