342 static int init_implementation_adapter_regs(struct cxl *adapter, struct pci_dev *dev)  in init_implementation_adapter_regs()  argument
361 	cxl_p1_write(adapter, CXL_PSL_DSNDCTL, psl_dsnctl);  in init_implementation_adapter_regs()
362 	cxl_p1_write(adapter, CXL_PSL_RESLCKTO, 0x20000000200ULL);  in init_implementation_adapter_regs()
364 	cxl_p1_write(adapter, CXL_PSL_SNWRALLOC, 0x00000000FFFFFFFFULL);  in init_implementation_adapter_regs()
366 	cxl_p1_write(adapter, CXL_PSL_FIR_CNTL, 0x0800000000000000ULL);  in init_implementation_adapter_regs()
368 	cxl_p1_write(adapter, CXL_PSL_TRACE, 0x0000FF7C00000000ULL);  in init_implementation_adapter_regs()
376 static int cxl_setup_psl_timebase(struct cxl *adapter, struct pci_dev *dev)  in cxl_setup_psl_timebase()  argument
399 	cxl_p1_write(adapter, CXL_PSL_TB_CTLSTAT,  in cxl_setup_psl_timebase()
403 	cxl_p1_write(adapter, CXL_PSL_Control, 0x0000000000000000);  in cxl_setup_psl_timebase()
404 	cxl_p1_write(adapter, CXL_PSL_Control, CXL_PSL_Control_tb);  in cxl_setup_psl_timebase()
413 		psl_tb = cxl_p1_read(adapter, CXL_PSL_Timebase);  in cxl_setup_psl_timebase()
435 int cxl_setup_irq(struct cxl *adapter, unsigned int hwirq,  in cxl_setup_irq()  argument
438 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_setup_irq()
443 int cxl_update_image_control(struct cxl *adapter)  in cxl_update_image_control()  argument
445 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_update_image_control()
460 	if (adapter->perst_loads_image)  in cxl_update_image_control()
465 	if (adapter->perst_select_user)  in cxl_update_image_control()
478 int cxl_alloc_one_irq(struct cxl *adapter)  in cxl_alloc_one_irq()  argument
480 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_alloc_one_irq()
485 void cxl_release_one_irq(struct cxl *adapter, int hwirq)  in cxl_release_one_irq()  argument
487 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_release_one_irq()
492 int cxl_alloc_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter, unsigned int num)  in cxl_alloc_irq_ranges()  argument
494 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_alloc_irq_ranges()
499 void cxl_release_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter)  in cxl_release_irq_ranges()  argument
501 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_release_irq_ranges()
560 static int cxl_map_slice_regs(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)  in cxl_map_slice_regs()  argument
568 	afu->psn_phys = p2_base(dev) + (adapter->ps_off + (afu->slice * adapter->ps_size));  in cxl_map_slice_regs()
569 	afu_desc = p2_base(dev) + adapter->afu_desc_off + (afu->slice * adapter->afu_desc_size);  in cxl_map_slice_regs()
576 		if (!(afu->afu_desc_mmio = ioremap(afu_desc, adapter->afu_desc_size)))  in cxl_map_slice_regs()
618 static struct cxl_afu *cxl_alloc_afu(struct cxl *adapter, int slice)  in cxl_alloc_afu()  argument
625 	afu->adapter = adapter;  in cxl_alloc_afu()
626 	afu->dev.parent = &adapter->dev;  in cxl_alloc_afu()
635 	afu->irqs_max = afu->adapter->user_irqs;  in cxl_alloc_afu()
690 	if (afu->psa && afu->adapter->ps_size <  in cxl_afu_descriptor_looks_ok()
806 static int cxl_configure_afu(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)  in cxl_configure_afu()  argument
810 	if ((rc = cxl_map_slice_regs(afu, adapter, dev)))  in cxl_configure_afu()
854 static int cxl_init_afu(struct cxl *adapter, int slice, struct pci_dev *dev)  in cxl_init_afu()  argument
859 	afu = cxl_alloc_afu(adapter, slice);  in cxl_init_afu()
863 	rc = dev_set_name(&afu->dev, "afu%i.%i", adapter->adapter_num, slice);  in cxl_init_afu()
867 	rc = cxl_configure_afu(afu, adapter, dev);  in cxl_init_afu()
884 	adapter->afu[afu->slice] = afu;  in cxl_init_afu()
913 	spin_lock(&afu->adapter->afu_list_lock);  in cxl_remove_afu()
914 	afu->adapter->afu[afu->slice] = NULL;  in cxl_remove_afu()
915 	spin_unlock(&afu->adapter->afu_list_lock);  in cxl_remove_afu()
924 int cxl_reset(struct cxl *adapter)  in cxl_reset()  argument
926 	struct pci_dev *dev = to_pci_dev(adapter->dev.parent);  in cxl_reset()
929 	if (adapter->perst_same_image) {  in cxl_reset()
948 static int cxl_map_adapter_regs(struct cxl *adapter, struct pci_dev *dev)  in cxl_map_adapter_regs()  argument
958 	if (!(adapter->p1_mmio = ioremap(p1_base(dev), p1_size(dev))))  in cxl_map_adapter_regs()
961 	if (!(adapter->p2_mmio = ioremap(p2_base(dev), p2_size(dev))))  in cxl_map_adapter_regs()
967 	iounmap(adapter->p1_mmio);  in cxl_map_adapter_regs()
968 	adapter->p1_mmio = NULL;  in cxl_map_adapter_regs()
977 static void cxl_unmap_adapter_regs(struct cxl *adapter)  in cxl_unmap_adapter_regs()  argument
979 	if (adapter->p1_mmio) {  in cxl_unmap_adapter_regs()
980 		iounmap(adapter->p1_mmio);  in cxl_unmap_adapter_regs()
981 		adapter->p1_mmio = NULL;  in cxl_unmap_adapter_regs()
982 		pci_release_region(to_pci_dev(adapter->dev.parent), 2);  in cxl_unmap_adapter_regs()
984 	if (adapter->p2_mmio) {  in cxl_unmap_adapter_regs()
985 		iounmap(adapter->p2_mmio);  in cxl_unmap_adapter_regs()
986 		adapter->p2_mmio = NULL;  in cxl_unmap_adapter_regs()
987 		pci_release_region(to_pci_dev(adapter->dev.parent), 0);  in cxl_unmap_adapter_regs()
991 static int cxl_read_vsec(struct cxl *adapter, struct pci_dev *dev)  in cxl_read_vsec()  argument
1010 	CXL_READ_VSEC_STATUS(dev, vsec, &adapter->vsec_status);  in cxl_read_vsec()
1011 	CXL_READ_VSEC_PSL_REVISION(dev, vsec, &adapter->psl_rev);  in cxl_read_vsec()
1012 	CXL_READ_VSEC_CAIA_MAJOR(dev, vsec, &adapter->caia_major);  in cxl_read_vsec()
1013 	CXL_READ_VSEC_CAIA_MINOR(dev, vsec, &adapter->caia_minor);  in cxl_read_vsec()
1014 	CXL_READ_VSEC_BASE_IMAGE(dev, vsec, &adapter->base_image);  in cxl_read_vsec()
1016 	adapter->user_image_loaded = !!(image_state & CXL_VSEC_USER_IMAGE_LOADED);  in cxl_read_vsec()
1017 	adapter->perst_select_user = !!(image_state & CXL_VSEC_USER_IMAGE_LOADED);  in cxl_read_vsec()
1019 	CXL_READ_VSEC_NAFUS(dev, vsec, &adapter->slices);  in cxl_read_vsec()
1027 	adapter->ps_off = ps_off * 64 * 1024;  in cxl_read_vsec()
1028 	adapter->ps_size = ps_size * 64 * 1024;  in cxl_read_vsec()
1029 	adapter->afu_desc_off = afu_desc_off * 64 * 1024;  in cxl_read_vsec()
1030 	adapter->afu_desc_size = afu_desc_size *64 * 1024;  in cxl_read_vsec()
1033 	adapter->user_irqs = pnv_cxl_get_irq_count(dev) - 1 - 2*adapter->slices;  in cxl_read_vsec()
1046 static void cxl_fixup_malformed_tlp(struct cxl *adapter, struct pci_dev *dev)  in cxl_fixup_malformed_tlp()  argument
1051 	if (adapter->psl_rev & 0xf000)  in cxl_fixup_malformed_tlp()
1064 static int cxl_vsec_looks_ok(struct cxl *adapter, struct pci_dev *dev)  in cxl_vsec_looks_ok()  argument
1066 	if (adapter->vsec_status & CXL_STATUS_SECOND_PORT)  in cxl_vsec_looks_ok()
1069 	if (adapter->vsec_status & CXL_UNSUPPORTED_FEATURES) {  in cxl_vsec_looks_ok()
1074 	if (!adapter->slices) {  in cxl_vsec_looks_ok()
1081 	if (!adapter->afu_desc_off || !adapter->afu_desc_size) {  in cxl_vsec_looks_ok()
1086 	if (adapter->ps_size > p2_size(dev) - adapter->ps_off) {  in cxl_vsec_looks_ok()
1089 			 adapter->ps_size, p2_size(dev) - adapter->ps_off);  in cxl_vsec_looks_ok()
1098 	struct cxl *adapter = to_cxl_adapter(dev);  in cxl_release_adapter()  local
1102 	cxl_remove_adapter_nr(adapter);  in cxl_release_adapter()
1104 	kfree(adapter);  in cxl_release_adapter()
1109 	struct cxl *adapter;  in cxl_alloc_adapter()  local
1111 	if (!(adapter = kzalloc(sizeof(struct cxl), GFP_KERNEL)))  in cxl_alloc_adapter()
1114 	spin_lock_init(&adapter->afu_list_lock);  in cxl_alloc_adapter()
1116 	if (cxl_alloc_adapter_nr(adapter))  in cxl_alloc_adapter()
1119 	if (dev_set_name(&adapter->dev, "card%i", adapter->adapter_num))  in cxl_alloc_adapter()
1122 	return adapter;  in cxl_alloc_adapter()
1125 	cxl_remove_adapter_nr(adapter);  in cxl_alloc_adapter()
1127 	kfree(adapter);  in cxl_alloc_adapter()
1133 static int sanitise_adapter_regs(struct cxl *adapter)  in sanitise_adapter_regs()  argument
1136 	cxl_p1_write(adapter, CXL_PSL_ErrIVTE, CXL_PSL_ErrIVTE_tberror);  in sanitise_adapter_regs()
1137 	return cxl_tlb_slb_invalidate(adapter);  in sanitise_adapter_regs()
1143 static int cxl_configure_adapter(struct cxl *adapter, struct pci_dev *dev)  in cxl_configure_adapter()  argument
1147 	adapter->dev.parent = &dev->dev;  in cxl_configure_adapter()
1148 	adapter->dev.release = cxl_release_adapter;  in cxl_configure_adapter()
1149 	pci_set_drvdata(dev, adapter);  in cxl_configure_adapter()
1157 	if ((rc = cxl_read_vsec(adapter, dev)))  in cxl_configure_adapter()
1160 	if ((rc = cxl_vsec_looks_ok(adapter, dev)))  in cxl_configure_adapter()
1163 	cxl_fixup_malformed_tlp(adapter, dev);  in cxl_configure_adapter()
1171 	if ((rc = cxl_update_image_control(adapter)))  in cxl_configure_adapter()
1174 	if ((rc = cxl_map_adapter_regs(adapter, dev)))  in cxl_configure_adapter()
1177 	if ((rc = sanitise_adapter_regs(adapter)))  in cxl_configure_adapter()
1180 	if ((rc = init_implementation_adapter_regs(adapter, dev)))  in cxl_configure_adapter()
1191 	if ((rc = cxl_setup_psl_timebase(adapter, dev)))  in cxl_configure_adapter()
1194 	if ((rc = cxl_register_psl_err_irq(adapter)))  in cxl_configure_adapter()
1200 	cxl_unmap_adapter_regs(adapter);  in cxl_configure_adapter()
1205 static void cxl_deconfigure_adapter(struct cxl *adapter)  in cxl_deconfigure_adapter()  argument
1207 	struct pci_dev *pdev = to_pci_dev(adapter->dev.parent);  in cxl_deconfigure_adapter()
1209 	cxl_release_psl_err_irq(adapter);  in cxl_deconfigure_adapter()
1210 	cxl_unmap_adapter_regs(adapter);  in cxl_deconfigure_adapter()
1217 	struct cxl *adapter;  in cxl_init_adapter()  local
1220 	adapter = cxl_alloc_adapter();  in cxl_init_adapter()
1221 	if (!adapter)  in cxl_init_adapter()
1227 	adapter->perst_loads_image = true;  in cxl_init_adapter()
1228 	adapter->perst_same_image = false;  in cxl_init_adapter()
1230 	rc = cxl_configure_adapter(adapter, dev);  in cxl_init_adapter()
1233 		cxl_release_adapter(&adapter->dev);  in cxl_init_adapter()
1238 	cxl_debugfs_adapter_add(adapter);  in cxl_init_adapter()
1244 	if ((rc = cxl_register_adapter(adapter)))  in cxl_init_adapter()
1247 	if ((rc = cxl_sysfs_adapter_add(adapter)))  in cxl_init_adapter()
1250 	return adapter;  in cxl_init_adapter()
1256 	cxl_debugfs_adapter_remove(adapter);  in cxl_init_adapter()
1257 	cxl_deconfigure_adapter(adapter);  in cxl_init_adapter()
1258 	device_unregister(&adapter->dev);  in cxl_init_adapter()
1262 static void cxl_remove_adapter(struct cxl *adapter)  in cxl_remove_adapter()  argument
1266 	cxl_sysfs_adapter_remove(adapter);  in cxl_remove_adapter()
1267 	cxl_debugfs_adapter_remove(adapter);  in cxl_remove_adapter()
1269 	cxl_deconfigure_adapter(adapter);  in cxl_remove_adapter()
1271 	device_unregister(&adapter->dev);  in cxl_remove_adapter()
1276 	struct cxl *adapter;  in cxl_probe()  local
1283 	adapter = cxl_init_adapter(dev);  in cxl_probe()
1284 	if (IS_ERR(adapter)) {  in cxl_probe()
1285 		dev_err(&dev->dev, "cxl_init_adapter failed: %li\n", PTR_ERR(adapter));  in cxl_probe()
1286 		return PTR_ERR(adapter);  in cxl_probe()
1289 	for (slice = 0; slice < adapter->slices; slice++) {  in cxl_probe()
1290 		if ((rc = cxl_init_afu(adapter, slice, dev))) {  in cxl_probe()
1295 		rc = cxl_afu_select_best_mode(adapter->afu[slice]);  in cxl_probe()
1305 	struct cxl *adapter = pci_get_drvdata(dev);  in cxl_remove()  local
1313 	for (i = 0; i < adapter->slices; i++) {  in cxl_remove()
1314 		afu = adapter->afu[i];  in cxl_remove()
1318 	cxl_remove_adapter(adapter);  in cxl_remove()
1353 	struct cxl *adapter = pci_get_drvdata(pdev);  in cxl_pci_error_detected()  local
1369 		for (i = 0; i < adapter->slices; i++) {  in cxl_pci_error_detected()
1370 			afu = adapter->afu[i];  in cxl_pci_error_detected()
1398 	if (adapter->perst_loads_image && !adapter->perst_same_image) {  in cxl_pci_error_detected()
1454 	for (i = 0; i < adapter->slices; i++) {  in cxl_pci_error_detected()
1455 		afu = adapter->afu[i];  in cxl_pci_error_detected()
1467 	cxl_deconfigure_adapter(adapter);  in cxl_pci_error_detected()
1474 	struct cxl *adapter = pci_get_drvdata(pdev);  in cxl_pci_slot_reset()  local
1482 	if (cxl_configure_adapter(adapter, pdev))  in cxl_pci_slot_reset()
1485 	for (i = 0; i < adapter->slices; i++) {  in cxl_pci_slot_reset()
1486 		afu = adapter->afu[i];  in cxl_pci_slot_reset()
1488 		if (cxl_configure_afu(afu, adapter, pdev))  in cxl_pci_slot_reset()
1546 	struct cxl *adapter = pci_get_drvdata(pdev);  in cxl_pci_resume()  local
1555 	for (i = 0; i < adapter->slices; i++) {  in cxl_pci_resume()
1556 		afu = adapter->afu[i];  in cxl_pci_resume()