112 static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);
121 void mce_setup(struct mce *m)  in mce_setup()  argument
123 	memset(m, 0, sizeof(struct mce));  in mce_setup()
124 	m->cpu = m->extcpu = smp_processor_id();  in mce_setup()
125 	rdtscll(m->tsc);  in mce_setup()
127 	m->time = get_seconds();  in mce_setup()
128 	m->cpuvendor = boot_cpu_data.x86_vendor;  in mce_setup()
129 	m->cpuid = cpuid_eax(1);  in mce_setup()
130 	m->socketid = cpu_data(m->extcpu).phys_proc_id;  in mce_setup()
131 	m->apicid = cpu_data(m->extcpu).initial_apicid;  in mce_setup()
132 	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);  in mce_setup()
203 		struct mce *m;  in drain_mcelog_buffer()  local
210 			m = &mcelog.entry[i];  in drain_mcelog_buffer()
212 			while (!m->finished) {  in drain_mcelog_buffer()
218 				if (!m->finished && retries >= 4) {  in drain_mcelog_buffer()
224 			atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, m);  in drain_mcelog_buffer()
227 		memset(mcelog.entry + prev, 0, (next - prev) * sizeof(*m));  in drain_mcelog_buffer()
247 static void print_mce(struct mce *m)  in print_mce()  argument
252 	       m->extcpu, m->mcgstatus, m->bank, m->status);  in print_mce()
254 	if (m->ip) {  in print_mce()
256 			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",  in print_mce()
257 				m->cs, m->ip);  in print_mce()
259 		if (m->cs == __KERNEL_CS)  in print_mce()
260 			print_symbol("{%s}", m->ip);  in print_mce()
264 	pr_emerg(HW_ERR "TSC %llx ", m->tsc);  in print_mce()
265 	if (m->addr)  in print_mce()
266 		pr_cont("ADDR %llx ", m->addr);  in print_mce()
267 	if (m->misc)  in print_mce()
268 		pr_cont("MISC %llx ", m->misc);  in print_mce()
276 		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,  in print_mce()
277 		cpu_data(m->extcpu).microcode);  in print_mce()
283 	ret = atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, m);  in print_mce()
332 		struct mce *m = &mcelog.entry[i];  in mce_panic()  local
333 		if (!(m->status & MCI_STATUS_VAL))  in mce_panic()
335 		if (!(m->status & MCI_STATUS_UC)) {  in mce_panic()
336 			print_mce(m);  in mce_panic()
338 				apei_err = apei_write_mce(m);  in mce_panic()
343 		struct mce *m = &mcelog.entry[i];  in mce_panic()  local
344 		if (!(m->status & MCI_STATUS_VAL))  in mce_panic()
346 		if (!(m->status & MCI_STATUS_UC))  in mce_panic()
348 		if (!final || memcmp(m, final, sizeof(struct mce))) {  in mce_panic()
349 			print_mce(m);  in mce_panic()
351 				apei_err = apei_write_mce(m);  in mce_panic()
433 static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)  in mce_gather_info()  argument
435 	mce_setup(m);  in mce_gather_info()
437 	m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);  in mce_gather_info()
443 		if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {  in mce_gather_info()
444 			m->ip = regs->ip;  in mce_gather_info()
445 			m->cs = regs->cs;  in mce_gather_info()
453 				m->cs |= 3;  in mce_gather_info()
457 			m->ip = mce_rdmsrl(mca_cfg.rip_msr);  in mce_gather_info()
557 static void mce_read_aux(struct mce *m, int i)  in mce_read_aux()  argument
559 	if (m->status & MCI_STATUS_MISCV)  in mce_read_aux()
560 		m->misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));  in mce_read_aux()
561 	if (m->status & MCI_STATUS_ADDRV) {  in mce_read_aux()
562 		m->addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));  in mce_read_aux()
567 		if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {  in mce_read_aux()
568 			u8 shift = MCI_MISC_ADDR_LSB(m->misc);  in mce_read_aux()
569 			m->addr >>= shift;  in mce_read_aux()
570 			m->addr <<= shift;  in mce_read_aux()
575 static bool memory_error(struct mce *m)  in memory_error()  argument
598 		return (m->status & 0xef80) == BIT(7) ||  in memory_error()
599 		       (m->status & 0xef00) == BIT(8) ||  in memory_error()
600 		       (m->status & 0xeffc) == 0xc;  in memory_error()
626 	struct mce m;  in machine_check_poll()  local
632 	mce_gather_info(&m, NULL);  in machine_check_poll()
638 		m.misc = 0;  in machine_check_poll()
639 		m.addr = 0;  in machine_check_poll()
640 		m.bank = i;  in machine_check_poll()
641 		m.tsc = 0;  in machine_check_poll()
644 		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));  in machine_check_poll()
645 		if (!(m.status & MCI_STATUS_VAL))  in machine_check_poll()
656 		    (m.status & (mca_cfg.ser ? MCI_STATUS_S : MCI_STATUS_UC)))  in machine_check_poll()
659 		mce_read_aux(&m, i);  in machine_check_poll()
662 			m.tsc = 0;  in machine_check_poll()
664 		severity = mce_severity(&m, mca_cfg.tolerant, NULL, false);  in machine_check_poll()
670 		if (severity == MCE_DEFERRED_SEVERITY && memory_error(&m)) {  in machine_check_poll()
671 			if (m.status & MCI_STATUS_ADDRV) {  in machine_check_poll()
672 				mce_ring_add(m.addr >> PAGE_SHIFT);  in machine_check_poll()
683 			mce_log(&m);  in machine_check_poll()
707 static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,  in mce_no_way_out()  argument
714 		m->status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));  in mce_no_way_out()
715 		if (m->status & MCI_STATUS_VAL) {  in mce_no_way_out()
718 				quirk_no_way_out(i, m, regs);  in mce_no_way_out()
721 		if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {  in mce_no_way_out()
795 	struct mce *m = NULL;  in mce_reign()  local
812 			m = &per_cpu(mces_seen, cpu);  in mce_reign()
821 	if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)  in mce_reign()
822 		mce_panic("Fatal machine check", m, msg);  in mce_reign()
992 static int mce_usable_address(struct mce *m)  in mce_usable_address()  argument
994 	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))  in mce_usable_address()
996 	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)  in mce_usable_address()
998 	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)  in mce_usable_address()
1028 	struct mce m, *final;  in do_machine_check()  local
1061 	mce_gather_info(&m, regs);  in do_machine_check()
1064 	*final = m;  in do_machine_check()
1067 	no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);  in do_machine_check()
1076 	if (!(m.mcgstatus & MCG_STATUS_RIPV))  in do_machine_check()
1092 		m.misc = 0;  in do_machine_check()
1093 		m.addr = 0;  in do_machine_check()
1094 		m.bank = i;  in do_machine_check()
1096 		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));  in do_machine_check()
1097 		if ((m.status & MCI_STATUS_VAL) == 0)  in do_machine_check()
1104 		if (!(m.status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&  in do_machine_check()
1113 		severity = mce_severity(&m, cfg->tolerant, NULL, true);  in do_machine_check()
1131 		mce_read_aux(&m, i);  in do_machine_check()
1140 		if (severity == MCE_AO_SEVERITY && mce_usable_address(&m))  in do_machine_check()
1141 			mce_ring_add(m.addr >> PAGE_SHIFT);  in do_machine_check()
1143 		mce_log(&m);  in do_machine_check()
1146 			*final = m;  in do_machine_check()
1152 	m = *final;  in do_machine_check()
1172 			mce_panic("Fatal machine check on current CPU", &m, msg);  in do_machine_check()
1174 			recover_paddr = m.addr;  in do_machine_check()
1175 			if (!(m.mcgstatus & MCG_STATUS_RIPV))  in do_machine_check()
1253 	struct mce m;  in mce_log_therm_throt_event()  local
1255 	mce_setup(&m);  in mce_log_therm_throt_event()
1256 	m.bank = MCE_THERMAL_BANK;  in mce_log_therm_throt_event()
1257 	m.status = status;  in mce_log_therm_throt_event()
1258 	mce_log(&m);  in mce_log_therm_throt_event()
1485 static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)  in quirk_sandybridge_ifu()  argument
1489 	if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)  in quirk_sandybridge_ifu()
1491 	if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|  in quirk_sandybridge_ifu()
1500 	m->mcgstatus |= MCG_STATUS_EIPV;  in quirk_sandybridge_ifu()
1501 	m->ip = regs->ip;  in quirk_sandybridge_ifu()
1502 	m->cs = regs->cs;  in quirk_sandybridge_ifu()
1768 	struct mce m;  in __mce_read_apei()  local
1773 	rc = apei_read_mce(&m, &record_id);  in __mce_read_apei()
1786 	if (copy_to_user(*ubuf, &m, sizeof(struct mce)))  in __mce_read_apei()
1836 			struct mce *m = &mcelog.entry[i];  in mce_chrdev_read()  local
1838 			while (!m->finished) {  in mce_chrdev_read()
1840 					memset(m, 0, sizeof(*m));  in mce_chrdev_read()
1846 			err |= copy_to_user(buf, m, sizeof(*m));  in mce_chrdev_read()
1847 			buf += sizeof(*m);  in mce_chrdev_read()
1867 		struct mce *m = &mcelog.entry[i];  in mce_chrdev_read()  local
1869 		if (m->finished && m->tsc < cpu_tsc[m->cpu]) {  in mce_chrdev_read()
1870 			err |= copy_to_user(buf, m, sizeof(*m));  in mce_chrdev_read()
1872 			buf += sizeof(*m);  in mce_chrdev_read()
1873 			memset(m, 0, sizeof(*m));  in mce_chrdev_read()