53 	struct mem_ctl_info *mci = dimm->mci;  in edac_dimm_info_location()  local
57 	for (i = 0; i < mci->n_layers; i++) {  in edac_dimm_info_location()
59 			      edac_layer_name[mci->layers[i].type],  in edac_dimm_info_location()
88 		 dimm->mci->csbased ? "rank" : "dimm",  in edac_mc_dump_dimm()
106 	edac_dbg(4, "  csrow->mci = %p\n", csrow->mci);  in edac_mc_dump_csrow()
109 static void edac_mc_dump_mci(struct mem_ctl_info *mci)  in edac_mc_dump_mci()  argument
111 	edac_dbg(3, "\tmci = %p\n", mci);  in edac_mc_dump_mci()
112 	edac_dbg(3, "\tmci->mtype_cap = %lx\n", mci->mtype_cap);  in edac_mc_dump_mci()
113 	edac_dbg(3, "\tmci->edac_ctl_cap = %lx\n", mci->edac_ctl_cap);  in edac_mc_dump_mci()
114 	edac_dbg(3, "\tmci->edac_cap = %lx\n", mci->edac_cap);  in edac_mc_dump_mci()
115 	edac_dbg(4, "\tmci->edac_check = %p\n", mci->edac_check);  in edac_mc_dump_mci()
117 		 mci->nr_csrows, mci->csrows);  in edac_mc_dump_mci()
119 		 mci->tot_dimms, mci->dimms);  in edac_mc_dump_mci()
120 	edac_dbg(3, "\tdev = %p\n", mci->pdev);  in edac_mc_dump_mci()
122 		 mci->mod_name, mci->ctl_name);  in edac_mc_dump_mci()
123 	edac_dbg(3, "\tpvt_info = %p\n\n", mci->pvt_info);  in edac_mc_dump_mci()
207 static void _edac_mc_free(struct mem_ctl_info *mci)  in _edac_mc_free()  argument
211 	const unsigned int tot_dimms = mci->tot_dimms;  in _edac_mc_free()
212 	const unsigned int tot_channels = mci->num_cschannel;  in _edac_mc_free()
213 	const unsigned int tot_csrows = mci->nr_csrows;  in _edac_mc_free()
215 	if (mci->dimms) {  in _edac_mc_free()
217 			kfree(mci->dimms[i]);  in _edac_mc_free()
218 		kfree(mci->dimms);  in _edac_mc_free()
220 	if (mci->csrows) {  in _edac_mc_free()
222 			csr = mci->csrows[row];  in _edac_mc_free()
232 		kfree(mci->csrows);  in _edac_mc_free()
234 	kfree(mci);  in _edac_mc_free()
266 	struct mem_ctl_info *mci;  in edac_mc_alloc()  local
300 	mci = edac_align_ptr(&ptr, sizeof(*mci), 1);  in edac_mc_alloc()
320 	mci = kzalloc(size, GFP_KERNEL);  in edac_mc_alloc()
321 	if (mci == NULL)  in edac_mc_alloc()
327 	layer = (struct edac_mc_layer *)(((char *)mci) + ((unsigned long)layer));  in edac_mc_alloc()
329 		mci->ce_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ce_per_layer[i]));  in edac_mc_alloc()
330 		mci->ue_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ue_per_layer[i]));  in edac_mc_alloc()
332 	pvt = sz_pvt ? (((char *)mci) + ((unsigned long)pvt)) : NULL;  in edac_mc_alloc()
335 	mci->mc_idx = mc_num;  in edac_mc_alloc()
336 	mci->tot_dimms = tot_dimms;  in edac_mc_alloc()
337 	mci->pvt_info = pvt;  in edac_mc_alloc()
338 	mci->n_layers = n_layers;  in edac_mc_alloc()
339 	mci->layers = layer;  in edac_mc_alloc()
340 	memcpy(mci->layers, layers, sizeof(*layer) * n_layers);  in edac_mc_alloc()
341 	mci->nr_csrows = tot_csrows;  in edac_mc_alloc()
342 	mci->num_cschannel = tot_channels;  in edac_mc_alloc()
343 	mci->csbased = per_rank;  in edac_mc_alloc()
348 	mci->csrows = kcalloc(tot_csrows, sizeof(*mci->csrows), GFP_KERNEL);  in edac_mc_alloc()
349 	if (!mci->csrows)  in edac_mc_alloc()
352 		csr = kzalloc(sizeof(**mci->csrows), GFP_KERNEL);  in edac_mc_alloc()
355 		mci->csrows[row] = csr;  in edac_mc_alloc()
357 		csr->mci = mci;  in edac_mc_alloc()
377 	mci->dimms  = kcalloc(tot_dimms, sizeof(*mci->dimms), GFP_KERNEL);  in edac_mc_alloc()
378 	if (!mci->dimms)  in edac_mc_alloc()
385 		chan = mci->csrows[row]->channels[chn];  in edac_mc_alloc()
388 …edac_mc_printk(mci, KERN_ERR, "EDAC core bug: EDAC_DIMM_OFF is trying to do an illegal data access…  in edac_mc_alloc()
392 		dimm = kzalloc(sizeof(**mci->dimms), GFP_KERNEL);  in edac_mc_alloc()
395 		mci->dimms[off] = dimm;  in edac_mc_alloc()
396 		dimm->mci = mci;  in edac_mc_alloc()
447 	mci->op_state = OP_ALLOC;  in edac_mc_alloc()
449 	return mci;  in edac_mc_alloc()
452 	_edac_mc_free(mci);  in edac_mc_alloc()
463 void edac_mc_free(struct mem_ctl_info *mci)  in edac_mc_free()  argument
470 	if (!device_is_registered(&mci->dev)) {  in edac_mc_free()
471 		_edac_mc_free(mci);  in edac_mc_free()
476 	edac_unregister_sysfs(mci);  in edac_mc_free()
490 	struct mem_ctl_info *mci;  in find_mci_by_dev()  local
496 		mci = list_entry(item, struct mem_ctl_info, link);  in find_mci_by_dev()
498 		if (mci->pdev == dev)  in find_mci_by_dev()
499 			return mci;  in find_mci_by_dev()
529 	struct mem_ctl_info *mci = to_edac_mem_ctl_work(d_work);  in edac_mc_workq_function()  local
534 	if (mci->op_state == OP_OFFLINE) {  in edac_mc_workq_function()
540 	if (edac_mc_assert_error_check_and_clear() && (mci->edac_check != NULL))  in edac_mc_workq_function()
541 		mci->edac_check(mci);  in edac_mc_workq_function()
546 	queue_delayed_work(edac_workqueue, &mci->work,  in edac_mc_workq_function()
559 static void edac_mc_workq_setup(struct mem_ctl_info *mci, unsigned msec,  in edac_mc_workq_setup()  argument
565 	if (mci->op_state != OP_RUNNING_POLL)  in edac_mc_workq_setup()
569 		INIT_DELAYED_WORK(&mci->work, edac_mc_workq_function);  in edac_mc_workq_setup()
571 	mod_delayed_work(edac_workqueue, &mci->work, msecs_to_jiffies(msec));  in edac_mc_workq_setup()
582 static void edac_mc_workq_teardown(struct mem_ctl_info *mci)  in edac_mc_workq_teardown()  argument
584 	mci->op_state = OP_OFFLINE;  in edac_mc_workq_teardown()
586 	cancel_delayed_work_sync(&mci->work);  in edac_mc_workq_teardown()
598 	struct mem_ctl_info *mci;  in edac_mc_reset_delay_period()  local
604 		mci = list_entry(item, struct mem_ctl_info, link);  in edac_mc_reset_delay_period()
606 		edac_mc_workq_setup(mci, value, false);  in edac_mc_reset_delay_period()
622 static int add_mc_to_global_list(struct mem_ctl_info *mci)  in add_mc_to_global_list()  argument
629 	p = find_mci_by_dev(mci->pdev);  in add_mc_to_global_list()
636 		if (p->mc_idx >= mci->mc_idx) {  in add_mc_to_global_list()
637 			if (unlikely(p->mc_idx == mci->mc_idx))  in add_mc_to_global_list()
645 	list_add_tail_rcu(&mci->link, insert_before);  in add_mc_to_global_list()
652 		edac_dev_name(mci), p->mod_name, p->ctl_name, p->mc_idx);  in add_mc_to_global_list()
662 static int del_mc_from_global_list(struct mem_ctl_info *mci)  in del_mc_from_global_list()  argument
665 	list_del_rcu(&mci->link);  in del_mc_from_global_list()
671 	INIT_LIST_HEAD(&mci->link);  in del_mc_from_global_list()
687 	struct mem_ctl_info *mci;  in edac_mc_find()  local
690 		mci = list_entry(item, struct mem_ctl_info, link);  in edac_mc_find()
692 		if (mci->mc_idx >= idx) {  in edac_mc_find()
693 			if (mci->mc_idx == idx)  in edac_mc_find()
694 				return mci;  in edac_mc_find()
716 int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,  in edac_mc_add_mc_with_groups()  argument
722 	if (mci->mc_idx >= EDAC_MAX_MCS) {  in edac_mc_add_mc_with_groups()
723 		pr_warn_once("Too many memory controllers: %d\n", mci->mc_idx);  in edac_mc_add_mc_with_groups()
729 		edac_mc_dump_mci(mci);  in edac_mc_add_mc_with_groups()
734 		for (i = 0; i < mci->nr_csrows; i++) {  in edac_mc_add_mc_with_groups()
735 			struct csrow_info *csrow = mci->csrows[i];  in edac_mc_add_mc_with_groups()
748 		for (i = 0; i < mci->tot_dimms; i++)  in edac_mc_add_mc_with_groups()
749 			if (mci->dimms[i]->nr_pages)  in edac_mc_add_mc_with_groups()
750 				edac_mc_dump_dimm(mci->dimms[i], i);  in edac_mc_add_mc_with_groups()
755 	if (edac_mc_owner && edac_mc_owner != mci->mod_name) {  in edac_mc_add_mc_with_groups()
760 	if (add_mc_to_global_list(mci))  in edac_mc_add_mc_with_groups()
764 	mci->start_time = jiffies;  in edac_mc_add_mc_with_groups()
766 	mci->bus = &mc_bus[mci->mc_idx];  in edac_mc_add_mc_with_groups()
768 	if (edac_create_sysfs_mci_device(mci, groups)) {  in edac_mc_add_mc_with_groups()
769 		edac_mc_printk(mci, KERN_WARNING,  in edac_mc_add_mc_with_groups()
775 	if (mci->edac_check != NULL) {  in edac_mc_add_mc_with_groups()
777 		mci->op_state = OP_RUNNING_POLL;  in edac_mc_add_mc_with_groups()
779 		edac_mc_workq_setup(mci, edac_mc_get_poll_msec(), true);  in edac_mc_add_mc_with_groups()
781 		mci->op_state = OP_RUNNING_INTERRUPT;  in edac_mc_add_mc_with_groups()
785 	edac_mc_printk(mci, KERN_INFO,  in edac_mc_add_mc_with_groups()
787 		mci->mod_name, mci->ctl_name, mci->dev_name,  in edac_mc_add_mc_with_groups()
788 		edac_op_state_to_string(mci->op_state));  in edac_mc_add_mc_with_groups()
790 	edac_mc_owner = mci->mod_name;  in edac_mc_add_mc_with_groups()
796 	del_mc_from_global_list(mci);  in edac_mc_add_mc_with_groups()
813 	struct mem_ctl_info *mci;  in edac_mc_del_mc()  local
820 	mci = find_mci_by_dev(dev);  in edac_mc_del_mc()
821 	if (mci == NULL) {  in edac_mc_del_mc()
826 	if (!del_mc_from_global_list(mci))  in edac_mc_del_mc()
831 	edac_mc_workq_teardown(mci);  in edac_mc_del_mc()
834 	mci->op_state = OP_OFFLINE;  in edac_mc_del_mc()
837 	edac_remove_sysfs_mci_device(mci);  in edac_mc_del_mc()
840 		"Removed device %d for %s %s: DEV %s\n", mci->mc_idx,  in edac_mc_del_mc()
841 		mci->mod_name, mci->ctl_name, edac_dev_name(mci));  in edac_mc_del_mc()
843 	return mci;  in edac_mc_del_mc()
879 int edac_mc_find_csrow_by_page(struct mem_ctl_info *mci, unsigned long page)  in edac_mc_find_csrow_by_page()  argument
881 	struct csrow_info **csrows = mci->csrows;  in edac_mc_find_csrow_by_page()
884 	edac_dbg(1, "MC%d: 0x%lx\n", mci->mc_idx, page);  in edac_mc_find_csrow_by_page()
887 	for (i = 0; i < mci->nr_csrows; i++) {  in edac_mc_find_csrow_by_page()
898 			 mci->mc_idx,  in edac_mc_find_csrow_by_page()
912 		edac_mc_printk(mci, KERN_ERR,  in edac_mc_find_csrow_by_page()
929 static void edac_inc_ce_error(struct mem_ctl_info *mci,  in edac_inc_ce_error()  argument
936 	mci->ce_mc += count;  in edac_inc_ce_error()
939 		mci->ce_noinfo_count += count;  in edac_inc_ce_error()
943 	for (i = 0; i < mci->n_layers; i++) {  in edac_inc_ce_error()
947 		mci->ce_per_layer[i][index] += count;  in edac_inc_ce_error()
949 		if (i < mci->n_layers - 1)  in edac_inc_ce_error()
950 			index *= mci->layers[i + 1].size;  in edac_inc_ce_error()
954 static void edac_inc_ue_error(struct mem_ctl_info *mci,  in edac_inc_ue_error()  argument
961 	mci->ue_mc += count;  in edac_inc_ue_error()
964 		mci->ce_noinfo_count += count;  in edac_inc_ue_error()
968 	for (i = 0; i < mci->n_layers; i++) {  in edac_inc_ue_error()
972 		mci->ue_per_layer[i][index] += count;  in edac_inc_ue_error()
974 		if (i < mci->n_layers - 1)  in edac_inc_ue_error()
975 			index *= mci->layers[i + 1].size;  in edac_inc_ue_error()
979 static void edac_ce_error(struct mem_ctl_info *mci,  in edac_ce_error()  argument
1000 			edac_mc_printk(mci, KERN_WARNING,  in edac_ce_error()
1005 			edac_mc_printk(mci, KERN_WARNING,  in edac_ce_error()
1010 	edac_inc_ce_error(mci, enable_per_layer_report, pos, error_count);  in edac_ce_error()
1012 	if (mci->scrub_mode == SCRUB_SW_SRC) {  in edac_ce_error()
1024 		remapped_page = mci->ctl_page_to_phys ?  in edac_ce_error()
1025 			mci->ctl_page_to_phys(mci, page_frame_number) :  in edac_ce_error()
1033 static void edac_ue_error(struct mem_ctl_info *mci,  in edac_ue_error()  argument
1050 			edac_mc_printk(mci, KERN_WARNING,  in edac_ue_error()
1055 			edac_mc_printk(mci, KERN_WARNING,  in edac_ue_error()
1070 	edac_inc_ue_error(mci, enable_per_layer_report, pos, error_count);  in edac_ue_error()
1086 			      struct mem_ctl_info *mci,  in edac_raw_mc_handle_error()  argument
1098 		edac_ce_error(mci, e->error_count, pos, e->msg, e->location, e->label,  in edac_raw_mc_handle_error()
1106 		edac_ue_error(mci, e->error_count, pos, e->msg, e->location, e->label,  in edac_raw_mc_handle_error()
1133 			  struct mem_ctl_info *mci,  in edac_mc_handle_error()  argument
1149 	struct edac_raw_error_desc *e = &mci->error_desc;  in edac_mc_handle_error()
1151 	edac_dbg(3, "MC%d\n", mci->mc_idx);  in edac_mc_handle_error()
1171 	for (i = 0; i < mci->n_layers; i++) {  in edac_mc_handle_error()
1172 		if (pos[i] >= (int)mci->layers[i].size) {  in edac_mc_handle_error()
1174 			edac_mc_printk(mci, KERN_ERR,  in edac_mc_handle_error()
1176 				       edac_layer_name[mci->layers[i].type],  in edac_mc_handle_error()
1177 				       pos[i], mci->layers[i].size);  in edac_mc_handle_error()
1204 	for (i = 0; i < mci->tot_dimms; i++) {  in edac_mc_handle_error()
1205 		struct dimm_info *dimm = mci->dimms[i];  in edac_mc_handle_error()
1243 				 mci->csbased ? "rank" : "dimm",  in edac_mc_handle_error()
1265 				mci->csrows[row]->ce_count += error_count;  in edac_mc_handle_error()
1267 					mci->csrows[row]->channels[chan]->ce_count += error_count;  in edac_mc_handle_error()
1271 				mci->csrows[row]->ue_count += error_count;  in edac_mc_handle_error()
1277 	for (i = 0; i < mci->n_layers; i++) {  in edac_mc_handle_error()
1282 			     edac_layer_name[mci->layers[i].type],  in edac_mc_handle_error()
1291 		       mci->mc_idx, e->top_layer, e->mid_layer, e->low_layer,  in edac_mc_handle_error()
1295 	edac_raw_mc_handle_error(type, mci, e);  in edac_mc_handle_error()