126 ioat_init_channel(struct ioatdma_device *ioat_dma,
128 static void ioat_intr_quirk(struct ioatdma_device *ioat_dma);
129 static int ioat_enumerate_channels(struct ioatdma_device *ioat_dma);
130 static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma);
309 static int ioat_dma_self_test(struct ioatdma_device *ioat_dma)  in ioat_dma_self_test()  argument
314 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_dma_self_test()
315 	struct device *dev = &ioat_dma->pdev->dev;  in ioat_dma_self_test()
358 	tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest,  in ioat_dma_self_test()
410 int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma)  in ioat_dma_setup_interrupts()  argument
413 	struct pci_dev *pdev = ioat_dma->pdev;  in ioat_dma_setup_interrupts()
431 	msixcnt = ioat_dma->dma_dev.chancnt;  in ioat_dma_setup_interrupts()
433 		ioat_dma->msix_entries[i].entry = i;  in ioat_dma_setup_interrupts()
435 	err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt);  in ioat_dma_setup_interrupts()
440 		msix = &ioat_dma->msix_entries[i];  in ioat_dma_setup_interrupts()
441 		ioat_chan = ioat_chan_by_index(ioat_dma, i);  in ioat_dma_setup_interrupts()
447 				msix = &ioat_dma->msix_entries[j];  in ioat_dma_setup_interrupts()
448 				ioat_chan = ioat_chan_by_index(ioat_dma, j);  in ioat_dma_setup_interrupts()
455 	ioat_dma->irq_mode = IOAT_MSIX;  in ioat_dma_setup_interrupts()
464 			       "ioat-msi", ioat_dma);  in ioat_dma_setup_interrupts()
469 	ioat_dma->irq_mode = IOAT_MSI;  in ioat_dma_setup_interrupts()
474 			       IRQF_SHARED, "ioat-intx", ioat_dma);  in ioat_dma_setup_interrupts()
478 	ioat_dma->irq_mode = IOAT_INTX;  in ioat_dma_setup_interrupts()
481 		ioat_intr_quirk(ioat_dma);  in ioat_dma_setup_interrupts()
483 	writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);  in ioat_dma_setup_interrupts()
488 	writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);  in ioat_dma_setup_interrupts()
489 	ioat_dma->irq_mode = IOAT_NOIRQ;  in ioat_dma_setup_interrupts()
494 static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma)  in ioat_disable_interrupts()  argument
497 	writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);  in ioat_disable_interrupts()
500 static int ioat_probe(struct ioatdma_device *ioat_dma)  in ioat_probe()  argument
503 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_probe()
504 	struct pci_dev *pdev = ioat_dma->pdev;  in ioat_probe()
508 	ioat_dma->dma_pool = pci_pool_create("dma_desc_pool", pdev,  in ioat_probe()
511 	if (!ioat_dma->dma_pool) {  in ioat_probe()
516 	ioat_dma->completion_pool = pci_pool_create("completion_pool", pdev,  in ioat_probe()
521 	if (!ioat_dma->completion_pool) {  in ioat_probe()
526 	ioat_enumerate_channels(ioat_dma);  in ioat_probe()
536 	err = ioat_dma_setup_interrupts(ioat_dma);  in ioat_probe()
540 	err = ioat3_dma_self_test(ioat_dma);  in ioat_probe()
547 	ioat_disable_interrupts(ioat_dma);  in ioat_probe()
549 	pci_pool_destroy(ioat_dma->completion_pool);  in ioat_probe()
551 	pci_pool_destroy(ioat_dma->dma_pool);  in ioat_probe()
556 static int ioat_register(struct ioatdma_device *ioat_dma)  in ioat_register()  argument
558 	int err = dma_async_device_register(&ioat_dma->dma_dev);  in ioat_register()
561 		ioat_disable_interrupts(ioat_dma);  in ioat_register()
562 		pci_pool_destroy(ioat_dma->completion_pool);  in ioat_register()
563 		pci_pool_destroy(ioat_dma->dma_pool);  in ioat_register()
569 static void ioat_dma_remove(struct ioatdma_device *ioat_dma)  in ioat_dma_remove()  argument
571 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_dma_remove()
573 	ioat_disable_interrupts(ioat_dma);  in ioat_dma_remove()
575 	ioat_kobject_del(ioat_dma);  in ioat_dma_remove()
579 	pci_pool_destroy(ioat_dma->dma_pool);  in ioat_dma_remove()
580 	pci_pool_destroy(ioat_dma->completion_pool);  in ioat_dma_remove()
589 static int ioat_enumerate_channels(struct ioatdma_device *ioat_dma)  in ioat_enumerate_channels()  argument
592 	struct device *dev = &ioat_dma->pdev->dev;  in ioat_enumerate_channels()
593 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_enumerate_channels()
598 	dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET);  in ioat_enumerate_channels()
600 	if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) {  in ioat_enumerate_channels()
602 			 dma->chancnt, ARRAY_SIZE(ioat_dma->idx));  in ioat_enumerate_channels()
603 		dma->chancnt = ARRAY_SIZE(ioat_dma->idx);  in ioat_enumerate_channels()
605 	xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET);  in ioat_enumerate_channels()
616 		ioat_init_channel(ioat_dma, ioat_chan, i);  in ioat_enumerate_channels()
635 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;  in ioat_free_chan_resources()  local
672 	pci_pool_free(ioat_dma->completion_pool, ioat_chan->completion,  in ioat_free_chan_resources()
704 		pci_pool_alloc(ioat_chan->ioat_dma->completion_pool,  in ioat_alloc_chan_resources()
752 ioat_init_channel(struct ioatdma_device *ioat_dma,  in ioat_init_channel()  argument
755 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_init_channel()
759 	ioat_chan->ioat_dma = ioat_dma;  in ioat_init_channel()
760 	ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));  in ioat_init_channel()
765 	ioat_dma->idx[idx] = ioat_chan;  in ioat_init_channel()
773 static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)  in ioat_xor_val_self_test()  argument
790 	struct device *dev = &ioat_dma->pdev->dev;  in ioat_xor_val_self_test()
791 	struct dma_device *dma = &ioat_dma->dma_dev;  in ioat_xor_val_self_test()
1040 static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma)  in ioat3_dma_self_test()  argument
1044 	rc = ioat_dma_self_test(ioat_dma);  in ioat3_dma_self_test()
1048 	rc = ioat_xor_val_self_test(ioat_dma);  in ioat3_dma_self_test()
1053 static void ioat_intr_quirk(struct ioatdma_device *ioat_dma)  in ioat_intr_quirk()  argument
1060 	dma = &ioat_dma->dma_dev;  in ioat_intr_quirk()
1066 	if (ioat_dma->cap & IOAT_CAP_DWBES) {  in ioat_intr_quirk()
1079 static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)  in ioat3_dma_probe()  argument
1081 	struct pci_dev *pdev = ioat_dma->pdev;  in ioat3_dma_probe()
1089 	dma = &ioat_dma->dma_dev;  in ioat3_dma_probe()
1098 	ioat_dma->cap = readl(ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET);  in ioat3_dma_probe()
1101 		ioat_dma->cap &=  in ioat3_dma_probe()
1105 	if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ)))  in ioat3_dma_probe()
1106 		ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);  in ioat3_dma_probe()
1108 	if (ioat_dma->cap & IOAT_CAP_XOR) {  in ioat3_dma_probe()
1119 	if (ioat_dma->cap & IOAT_CAP_PQ) {  in ioat3_dma_probe()
1127 		if (ioat_dma->cap & IOAT_CAP_RAID16SS)  in ioat3_dma_probe()
1132 		if (!(ioat_dma->cap & IOAT_CAP_XOR)) {  in ioat3_dma_probe()
1138 			if (ioat_dma->cap & IOAT_CAP_RAID16SS)  in ioat3_dma_probe()
1148 	if (ioat_dma->cap & IOAT_CAP_RAID16SS) {  in ioat3_dma_probe()
1156 			ioat_dma->sed_hw_pool[i] = dmam_pool_create(pool_name,  in ioat3_dma_probe()
1159 			if (!ioat_dma->sed_hw_pool[i])  in ioat3_dma_probe()
1165 	if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ)))  in ioat3_dma_probe()
1168 	err = ioat_probe(ioat_dma);  in ioat3_dma_probe()
1178 	err = ioat_register(ioat_dma);  in ioat3_dma_probe()
1182 	ioat_kobject_add(ioat_dma, &ioat_ktype);  in ioat3_dma_probe()
1185 		ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base);  in ioat3_dma_probe()
1192 	struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);  in ioat_shutdown()  local
1196 	if (!ioat_dma)  in ioat_shutdown()
1200 		ioat_chan = ioat_dma->idx[i];  in ioat_shutdown()
1212 	ioat_disable_interrupts(ioat_dma);  in ioat_shutdown()
1215 void ioat_resume(struct ioatdma_device *ioat_dma)  in ioat_resume()  argument
1222 		ioat_chan = ioat_dma->idx[i];  in ioat_resume()
1279 	struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);  in ioat_pcie_error_resume()  local
1284 	ioat_resume(ioat_dma);  in ioat_pcie_error_resume()