408 #define CH_ACTIVE(pvt, ch)	((pvt)->info.mc_control & (1 << (8 + ch)))  argument
409 #define ECCx8(pvt)		((pvt)->info.mc_control & (1 << 1))  argument
412 #define ECC_ENABLED(pvt)	((pvt)->info.mc_status & (1 << 4))  argument
413 #define CH_DISABLED(pvt, ch)	((pvt)->info.mc_status & (1 << ch))  argument
501 	struct i7core_pvt *pvt = mci->pvt_info;  in get_dimm_config()  local
509 	pdev = pvt->pci_mcr[0];  in get_dimm_config()
514 	pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);  in get_dimm_config()
515 	pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);  in get_dimm_config()
516 	pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);  in get_dimm_config()
517 	pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);  in get_dimm_config()
520 		 pvt->i7core_dev->socket, pvt->info.mc_control,  in get_dimm_config()
521 		 pvt->info.mc_status, pvt->info.max_dod, pvt->info.ch_map);  in get_dimm_config()
523 	if (ECC_ENABLED(pvt)) {  in get_dimm_config()
524 		edac_dbg(0, "ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);  in get_dimm_config()
525 		if (ECCx8(pvt))  in get_dimm_config()
536 		 numdimms(pvt->info.max_dod),  in get_dimm_config()
537 		 numrank(pvt->info.max_dod >> 2),  in get_dimm_config()
538 		 numbank(pvt->info.max_dod >> 4),  in get_dimm_config()
539 		 numrow(pvt->info.max_dod >> 6),  in get_dimm_config()
540 		 numcol(pvt->info.max_dod >> 9));  in get_dimm_config()
545 		if (!pvt->pci_ch[i][0])  in get_dimm_config()
548 		if (!CH_ACTIVE(pvt, i)) {  in get_dimm_config()
552 		if (CH_DISABLED(pvt, i)) {  in get_dimm_config()
558 		pci_read_config_dword(pvt->pci_ch[i][0],  in get_dimm_config()
563 			pvt->channel[i].is_3dimms_present = true;  in get_dimm_config()
566 			pvt->channel[i].is_single_4rank = true;  in get_dimm_config()
569 			pvt->channel[i].has_4rank = true;  in get_dimm_config()
577 		pci_read_config_dword(pvt->pci_ch[i][1],  in get_dimm_config()
579 		pci_read_config_dword(pvt->pci_ch[i][1],  in get_dimm_config()
581 		pci_read_config_dword(pvt->pci_ch[i][1],  in get_dimm_config()
586 			 RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),  in get_dimm_config()
588 			 pvt->channel[i].is_3dimms_present ? "3DIMMS " : "",  in get_dimm_config()
589 			 pvt->channel[i].is_3dimms_present ? "SINGLE_4R " : "",  in get_dimm_config()
590 			 pvt->channel[i].has_4rank ? "HAS_4R " : "",  in get_dimm_config()
635 				 pvt->i7core_dev->socket, i, j);  in get_dimm_config()
675 	struct i7core_pvt *pvt = mci->pvt_info;  in disable_inject()  local
677 	pvt->inject.enable = 0;  in disable_inject()
679 	if (!pvt->pci_ch[pvt->inject.channel][0])  in disable_inject()
682 	pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],  in disable_inject()
700 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_inject_section_store()  local
704 	if (pvt->inject.enable)  in i7core_inject_section_store()
711 	pvt->inject.section = (u32) value;  in i7core_inject_section_store()
720 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_inject_section_show()  local
721 	return sprintf(data, "0x%08x\n", pvt->inject.section);  in i7core_inject_section_show()
737 struct i7core_pvt *pvt = mci->pvt_info;  in i7core_inject_type_store()  local
741 	if (pvt->inject.enable)  in i7core_inject_type_store()
748 	pvt->inject.type = (u32) value;  in i7core_inject_type_store()
757 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_inject_type_show()  local
759 	return sprintf(data, "0x%08x\n", pvt->inject.type);  in i7core_inject_type_show()
777 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_inject_eccmask_store()  local
781 	if (pvt->inject.enable)  in i7core_inject_eccmask_store()
788 	pvt->inject.eccmask = (u32) value;  in i7core_inject_eccmask_store()
797 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_inject_eccmask_show()  local
799 	return sprintf(data, "0x%08x\n", pvt->inject.eccmask);  in i7core_inject_eccmask_show()
820 	struct i7core_pvt *pvt;					\
825 	pvt = mci->pvt_info;					\
827 	if (pvt->inject.enable)					\
838 	pvt->inject.param = value;				\
849 	struct i7core_pvt *pvt;					\
851 	pvt = mci->pvt_info;					\
852 	edac_dbg(1, "pvt=%p\n", pvt);				\
853 	if (pvt->inject.param < 0)				\
856 		return sprintf(data, "%d\n", pvt->inject.param);\
928 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_inject_enable_store()  local
934 	if (!pvt->pci_ch[pvt->inject.channel][0])  in i7core_inject_enable_store()
942 		pvt->inject.enable = 1;  in i7core_inject_enable_store()
949 	if (pvt->inject.dimm < 0)  in i7core_inject_enable_store()
952 		if (pvt->channel[pvt->inject.channel].dimms > 2)  in i7core_inject_enable_store()
953 			mask |= (pvt->inject.dimm & 0x3LL) << 35;  in i7core_inject_enable_store()
955 			mask |= (pvt->inject.dimm & 0x1LL) << 36;  in i7core_inject_enable_store()
959 	if (pvt->inject.rank < 0)  in i7core_inject_enable_store()
962 		if (pvt->channel[pvt->inject.channel].dimms > 2)  in i7core_inject_enable_store()
963 			mask |= (pvt->inject.rank & 0x1LL) << 34;  in i7core_inject_enable_store()
965 			mask |= (pvt->inject.rank & 0x3LL) << 34;  in i7core_inject_enable_store()
969 	if (pvt->inject.bank < 0)  in i7core_inject_enable_store()
972 		mask |= (pvt->inject.bank & 0x15LL) << 30;  in i7core_inject_enable_store()
975 	if (pvt->inject.page < 0)  in i7core_inject_enable_store()
978 		mask |= (pvt->inject.page & 0xffff) << 14;  in i7core_inject_enable_store()
981 	if (pvt->inject.col < 0)  in i7core_inject_enable_store()
984 		mask |= (pvt->inject.col & 0x3fff);  in i7core_inject_enable_store()
993 	injectmask = (pvt->inject.type & 1) |  in i7core_inject_enable_store()
994 		     (pvt->inject.section & 0x3) << 1 |  in i7core_inject_enable_store()
995 		     (pvt->inject.type & 0x6) << (3 - 1);  in i7core_inject_enable_store()
998 	pci_write_config_dword(pvt->pci_noncore,  in i7core_inject_enable_store()
1001 	write_and_test(pvt->pci_ch[pvt->inject.channel][0],  in i7core_inject_enable_store()
1003 	write_and_test(pvt->pci_ch[pvt->inject.channel][0],  in i7core_inject_enable_store()
1006 	write_and_test(pvt->pci_ch[pvt->inject.channel][0],  in i7core_inject_enable_store()
1007 			       MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);  in i7core_inject_enable_store()
1009 	write_and_test(pvt->pci_ch[pvt->inject.channel][0],  in i7core_inject_enable_store()
1017 	pci_write_config_dword(pvt->pci_noncore,  in i7core_inject_enable_store()
1021 		 mask, pvt->inject.eccmask, injectmask);  in i7core_inject_enable_store()
1032 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_inject_enable_show()  local
1035 	if (!pvt->pci_ch[pvt->inject.channel][0])  in i7core_inject_enable_show()
1038 	pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],  in i7core_inject_enable_show()
1044 		pvt->inject.enable = 1;  in i7core_inject_enable_show()
1046 	return sprintf(data, "%d\n", pvt->inject.enable);  in i7core_inject_enable_show()
1056 	struct i7core_pvt *pvt = mci->pvt_info;			\
1059 	if (!pvt->ce_count_available || (pvt->is_registered))	\
1062 			pvt->udimm_ce_count[param]);		\
1172 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_create_sysfs_devices()  local
1175 	pvt->addrmatch_dev = kzalloc(sizeof(*pvt->addrmatch_dev), GFP_KERNEL);  in i7core_create_sysfs_devices()
1176 	if (!pvt->addrmatch_dev)  in i7core_create_sysfs_devices()
1179 	pvt->addrmatch_dev->type = &addrmatch_type;  in i7core_create_sysfs_devices()
1180 	pvt->addrmatch_dev->bus = mci->dev.bus;  in i7core_create_sysfs_devices()
1181 	device_initialize(pvt->addrmatch_dev);  in i7core_create_sysfs_devices()
1182 	pvt->addrmatch_dev->parent = &mci->dev;  in i7core_create_sysfs_devices()
1183 	dev_set_name(pvt->addrmatch_dev, "inject_addrmatch");  in i7core_create_sysfs_devices()
1184 	dev_set_drvdata(pvt->addrmatch_dev, mci);  in i7core_create_sysfs_devices()
1186 	edac_dbg(1, "creating %s\n", dev_name(pvt->addrmatch_dev));  in i7core_create_sysfs_devices()
1188 	rc = device_add(pvt->addrmatch_dev);  in i7core_create_sysfs_devices()
1192 	if (!pvt->is_registered) {  in i7core_create_sysfs_devices()
1193 		pvt->chancounts_dev = kzalloc(sizeof(*pvt->chancounts_dev),  in i7core_create_sysfs_devices()
1195 		if (!pvt->chancounts_dev) {  in i7core_create_sysfs_devices()
1196 			put_device(pvt->addrmatch_dev);  in i7core_create_sysfs_devices()
1197 			device_del(pvt->addrmatch_dev);  in i7core_create_sysfs_devices()
1201 		pvt->chancounts_dev->type = &all_channel_counts_type;  in i7core_create_sysfs_devices()
1202 		pvt->chancounts_dev->bus = mci->dev.bus;  in i7core_create_sysfs_devices()
1203 		device_initialize(pvt->chancounts_dev);  in i7core_create_sysfs_devices()
1204 		pvt->chancounts_dev->parent = &mci->dev;  in i7core_create_sysfs_devices()
1205 		dev_set_name(pvt->chancounts_dev, "all_channel_counts");  in i7core_create_sysfs_devices()
1206 		dev_set_drvdata(pvt->chancounts_dev, mci);  in i7core_create_sysfs_devices()
1208 		edac_dbg(1, "creating %s\n", dev_name(pvt->chancounts_dev));  in i7core_create_sysfs_devices()
1210 		rc = device_add(pvt->chancounts_dev);  in i7core_create_sysfs_devices()
1219 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_delete_sysfs_devices()  local
1223 	if (!pvt->is_registered) {  in i7core_delete_sysfs_devices()
1224 		put_device(pvt->chancounts_dev);  in i7core_delete_sysfs_devices()
1225 		device_del(pvt->chancounts_dev);  in i7core_delete_sysfs_devices()
1227 	put_device(pvt->addrmatch_dev);  in i7core_delete_sysfs_devices()
1228 	device_del(pvt->addrmatch_dev);  in i7core_delete_sysfs_devices()
1460 	struct i7core_pvt *pvt = mci->pvt_info;  in mci_bind_devs()  local
1465 	pvt->is_registered = false;  in mci_bind_devs()
1466 	pvt->enable_scrub  = false;  in mci_bind_devs()
1477 			pvt->pci_mcr[func] = pdev;  in mci_bind_devs()
1481 			pvt->pci_ch[slot - 4][func] = pdev;  in mci_bind_devs()
1483 			pvt->pci_noncore = pdev;  in mci_bind_devs()
1489 				pvt->enable_scrub = false;  in mci_bind_devs()
1493 				pvt->enable_scrub = false;  in mci_bind_devs()
1497 				pvt->enable_scrub = false;  in mci_bind_devs()
1501 				pvt->enable_scrub = true;  in mci_bind_devs()
1505 				pvt->enable_scrub = true;  in mci_bind_devs()
1509 				pvt->enable_scrub = false;  in mci_bind_devs()
1521 			pvt->is_registered = true;  in mci_bind_devs()
1543 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_rdimm_update_ce_count()  local
1546 	if (pvt->ce_count_available) {  in i7core_rdimm_update_ce_count()
1549 		add2 = new2 - pvt->rdimm_last_ce_count[chan][2];  in i7core_rdimm_update_ce_count()
1550 		add1 = new1 - pvt->rdimm_last_ce_count[chan][1];  in i7core_rdimm_update_ce_count()
1551 		add0 = new0 - pvt->rdimm_last_ce_count[chan][0];  in i7core_rdimm_update_ce_count()
1555 		pvt->rdimm_ce_count[chan][2] += add2;  in i7core_rdimm_update_ce_count()
1559 		pvt->rdimm_ce_count[chan][1] += add1;  in i7core_rdimm_update_ce_count()
1563 		pvt->rdimm_ce_count[chan][0] += add0;  in i7core_rdimm_update_ce_count()
1565 		pvt->ce_count_available = 1;  in i7core_rdimm_update_ce_count()
1568 	pvt->rdimm_last_ce_count[chan][2] = new2;  in i7core_rdimm_update_ce_count()
1569 	pvt->rdimm_last_ce_count[chan][1] = new1;  in i7core_rdimm_update_ce_count()
1570 	pvt->rdimm_last_ce_count[chan][0] = new0;  in i7core_rdimm_update_ce_count()
1589 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_rdimm_check_mc_ecc_err()  local
1594 	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,  in i7core_rdimm_check_mc_ecc_err()
1596 	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,  in i7core_rdimm_check_mc_ecc_err()
1598 	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,  in i7core_rdimm_check_mc_ecc_err()
1600 	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,  in i7core_rdimm_check_mc_ecc_err()
1602 	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,  in i7core_rdimm_check_mc_ecc_err()
1604 	pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,  in i7core_rdimm_check_mc_ecc_err()
1610 		if (pvt->channel[i].dimms > 2) {  in i7core_rdimm_check_mc_ecc_err()
1634 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_udimm_check_mc_ecc_err()  local
1638 	if (!pvt->pci_mcr[4]) {  in i7core_udimm_check_mc_ecc_err()
1644 	pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);  in i7core_udimm_check_mc_ecc_err()
1645 	pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);  in i7core_udimm_check_mc_ecc_err()
1653 	if (pvt->ce_count_available) {  in i7core_udimm_check_mc_ecc_err()
1657 		add2 = new2 - pvt->udimm_last_ce_count[2];  in i7core_udimm_check_mc_ecc_err()
1658 		add1 = new1 - pvt->udimm_last_ce_count[1];  in i7core_udimm_check_mc_ecc_err()
1659 		add0 = new0 - pvt->udimm_last_ce_count[0];  in i7core_udimm_check_mc_ecc_err()
1663 		pvt->udimm_ce_count[2] += add2;  in i7core_udimm_check_mc_ecc_err()
1667 		pvt->udimm_ce_count[1] += add1;  in i7core_udimm_check_mc_ecc_err()
1671 		pvt->udimm_ce_count[0] += add0;  in i7core_udimm_check_mc_ecc_err()
1678 		pvt->ce_count_available = 1;  in i7core_udimm_check_mc_ecc_err()
1681 	pvt->udimm_last_ce_count[2] = new2;  in i7core_udimm_check_mc_ecc_err()
1682 	pvt->udimm_last_ce_count[1] = new1;  in i7core_udimm_check_mc_ecc_err()
1683 	pvt->udimm_last_ce_count[0] = new0;  in i7core_udimm_check_mc_ecc_err()
1702 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_mce_output_error()  local
1782 	if (uncorrected_error || !pvt->is_registered)  in i7core_mce_output_error()
1797 	struct i7core_pvt *pvt = mci->pvt_info;  in i7core_check_error()  local
1808 	count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)  in i7core_check_error()
1813 	m = pvt->mce_outentry;  in i7core_check_error()
1814 	if (pvt->mce_in + count > MCE_LOG_LEN) {  in i7core_check_error()
1815 		unsigned l = MCE_LOG_LEN - pvt->mce_in;  in i7core_check_error()
1817 		memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);  in i7core_check_error()
1819 		pvt->mce_in = 0;  in i7core_check_error()
1823 	memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);  in i7core_check_error()
1825 	pvt->mce_in += count;  in i7core_check_error()
1828 	if (pvt->mce_overrun) {  in i7core_check_error()
1830 			      pvt->mce_overrun);  in i7core_check_error()
1832 		pvt->mce_overrun = 0;  in i7core_check_error()
1839 		i7core_mce_output_error(mci, &pvt->mce_outentry[i]);  in i7core_check_error()
1845 	if (!pvt->is_registered)  in i7core_check_error()
1865 	struct i7core_pvt *pvt;  in i7core_mce_check_error()  local
1872 	pvt = mci->pvt_info;  in i7core_mce_check_error()
1886 	if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {  in i7core_mce_check_error()
1888 		pvt->mce_overrun++;  in i7core_mce_check_error()
1893 	memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));  in i7core_mce_check_error()
1895 	pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;  in i7core_mce_check_error()
2022 	struct i7core_pvt *pvt = mci->pvt_info;  in set_sdram_scrub_rate()  local
2028 	pdev = pvt->pci_mcr[2];  in set_sdram_scrub_rate()
2047 		const u32 freq_dclk_mhz = pvt->dclk_freq;  in set_sdram_scrub_rate()
2084 	struct i7core_pvt *pvt = mci->pvt_info;  in get_sdram_scrub_rate()  local
2087 	const u32 freq_dclk_mhz = pvt->dclk_freq;  in get_sdram_scrub_rate()
2092 	pdev = pvt->pci_mcr[2];  in get_sdram_scrub_rate()
2113 	struct i7core_pvt *pvt = mci->pvt_info;  in enable_sdram_scrub_setting()  local
2117 	pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);  in enable_sdram_scrub_setting()
2119 	pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,  in enable_sdram_scrub_setting()
2128 	struct i7core_pvt *pvt = mci->pvt_info;  in disable_sdram_scrub_setting()  local
2132 	pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);  in disable_sdram_scrub_setting()
2134 	pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,  in disable_sdram_scrub_setting()
2138 static void i7core_pci_ctl_create(struct i7core_pvt *pvt)  in i7core_pci_ctl_create()  argument
2140 	pvt->i7core_pci = edac_pci_create_generic_ctl(  in i7core_pci_ctl_create()
2141 						&pvt->i7core_dev->pdev[0]->dev,  in i7core_pci_ctl_create()
2143 	if (unlikely(!pvt->i7core_pci))  in i7core_pci_ctl_create()
2148 static void i7core_pci_ctl_release(struct i7core_pvt *pvt)  in i7core_pci_ctl_release()  argument
2150 	if (likely(pvt->i7core_pci))  in i7core_pci_ctl_release()
2151 		edac_pci_release_generic_ctl(pvt->i7core_pci);  in i7core_pci_ctl_release()
2155 				pvt->i7core_dev->socket);  in i7core_pci_ctl_release()
2156 	pvt->i7core_pci = NULL;  in i7core_pci_ctl_release()
2162 	struct i7core_pvt *pvt;  in i7core_unregister_mci()  local
2171 	pvt = mci->pvt_info;  in i7core_unregister_mci()
2176 	if (pvt->enable_scrub)  in i7core_unregister_mci()
2180 	i7core_pci_ctl_release(pvt);  in i7core_unregister_mci()
2195 	struct i7core_pvt *pvt;  in i7core_register_mci()  local
2208 			    sizeof(*pvt));  in i7core_register_mci()
2214 	pvt = mci->pvt_info;  in i7core_register_mci()
2215 	memset(pvt, 0, sizeof(*pvt));  in i7core_register_mci()
2218 	pvt->i7core_dev = i7core_dev;  in i7core_register_mci()
2250 	if (pvt->enable_scrub)  in i7core_register_mci()
2271 	pvt->inject.channel = 0;  in i7core_register_mci()
2272 	pvt->inject.dimm = -1;  in i7core_register_mci()
2273 	pvt->inject.rank = -1;  in i7core_register_mci()
2274 	pvt->inject.bank = -1;  in i7core_register_mci()
2275 	pvt->inject.page = -1;  in i7core_register_mci()
2276 	pvt->inject.col = -1;  in i7core_register_mci()
2279 	i7core_pci_ctl_create(pvt);  in i7core_register_mci()
2282 	pvt->dclk_freq = get_dclk_freq();  in i7core_register_mci()