100 void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx)  in ioat_init_channel()  argument
102 	struct dma_device *dma = &device->common;  in ioat_init_channel()
106 	chan->device = device;  in ioat_init_channel()
107 	chan->reg_base = device->reg_base + (0x80 * (idx + 1));  in ioat_init_channel()
109 	chan->common.device = dma;  in ioat_init_channel()
112 	device->idx[idx] = chan;  in ioat_init_channel()
114 	chan->timer.function = device->timer_fn;  in ioat_init_channel()
116 	tasklet_init(&chan->cleanup_task, device->cleanup_fn, data);  in ioat_init_channel()
123 static int ioat1_enumerate_channels(struct ioatdma_device *device)  in ioat1_enumerate_channels()  argument
129 	struct device *dev = &device->pdev->dev;  in ioat1_enumerate_channels()
130 	struct dma_device *dma = &device->common;  in ioat1_enumerate_channels()
133 	dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);  in ioat1_enumerate_channels()
135 	if (dma->chancnt > ARRAY_SIZE(device->idx)) {  in ioat1_enumerate_channels()
137 			 dma->chancnt, ARRAY_SIZE(device->idx));  in ioat1_enumerate_channels()
138 		dma->chancnt = ARRAY_SIZE(device->idx);  in ioat1_enumerate_channels()
140 	xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);  in ioat1_enumerate_channels()
154 		ioat_init_channel(device, &ioat->base, i);  in ioat1_enumerate_channels()
221 	       reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));  in ioat1_reset_channel()
276 	ioatdma_device = ioat->base.device;  in ioat_dma_alloc_descriptor()
346 	chan->completion = pci_pool_alloc(chan->device->completion_pool,  in ioat1_dma_alloc_chan_resources()
363 	struct ioatdma_device *device = chan->device;  in ioat_stop()  local
364 	struct pci_dev *pdev = device->pdev;  in ioat_stop()
374 	switch (device->irq_mode) {  in ioat_stop()
376 		msix = &device->msix_entries[chan_id];  in ioat_stop()
394 	device->cleanup_fn((unsigned long) &chan->common);  in ioat_stop()
405 	struct ioatdma_device *ioatdma_device = chan->device;  in ioat1_dma_free_chan_resources()
421 	       chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));  in ioat1_dma_free_chan_resources()
751 	struct ioatdma_device *device = chan->device;  in ioat_dma_tx_status()  local
758 	device->cleanup_fn((unsigned long) c);  in ioat_dma_tx_status()
815 int ioat_dma_self_test(struct ioatdma_device *device)  in ioat_dma_self_test()  argument
820 	struct dma_device *dma = &device->common;  in ioat_dma_self_test()
821 	struct device *dev = &device->pdev->dev;  in ioat_dma_self_test()
864 	tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,  in ioat_dma_self_test()
921 int ioat_dma_setup_interrupts(struct ioatdma_device *device)  in ioat_dma_setup_interrupts()  argument
924 	struct pci_dev *pdev = device->pdev;  in ioat_dma_setup_interrupts()
925 	struct device *dev = &pdev->dev;  in ioat_dma_setup_interrupts()
942 	msixcnt = device->common.chancnt;  in ioat_dma_setup_interrupts()
944 		device->msix_entries[i].entry = i;  in ioat_dma_setup_interrupts()
946 	err = pci_enable_msix_exact(pdev, device->msix_entries, msixcnt);  in ioat_dma_setup_interrupts()
951 		msix = &device->msix_entries[i];  in ioat_dma_setup_interrupts()
952 		chan = ioat_chan_by_index(device, i);  in ioat_dma_setup_interrupts()
958 				msix = &device->msix_entries[j];  in ioat_dma_setup_interrupts()
959 				chan = ioat_chan_by_index(device, j);  in ioat_dma_setup_interrupts()
966 	device->irq_mode = IOAT_MSIX;  in ioat_dma_setup_interrupts()
975 			       "ioat-msi", device);  in ioat_dma_setup_interrupts()
980 	device->irq_mode = IOAT_MSI;  in ioat_dma_setup_interrupts()
985 			       IRQF_SHARED, "ioat-intx", device);  in ioat_dma_setup_interrupts()
989 	device->irq_mode = IOAT_INTX;  in ioat_dma_setup_interrupts()
991 	if (device->intr_quirk)  in ioat_dma_setup_interrupts()
992 		device->intr_quirk(device);  in ioat_dma_setup_interrupts()
994 	writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);  in ioat_dma_setup_interrupts()
999 	writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);  in ioat_dma_setup_interrupts()
1000 	device->irq_mode = IOAT_NOIRQ;  in ioat_dma_setup_interrupts()
1006 static void ioat_disable_interrupts(struct ioatdma_device *device)  in ioat_disable_interrupts()  argument
1009 	writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);  in ioat_disable_interrupts()
1012 int ioat_probe(struct ioatdma_device *device)  in ioat_probe()  argument
1015 	struct dma_device *dma = &device->common;  in ioat_probe()
1016 	struct pci_dev *pdev = device->pdev;  in ioat_probe()
1017 	struct device *dev = &pdev->dev;  in ioat_probe()
1020 	device->dma_pool = pci_pool_create("dma_desc_pool", pdev,  in ioat_probe()
1023 	if (!device->dma_pool) {  in ioat_probe()
1028 	device->completion_pool = pci_pool_create("completion_pool", pdev,  in ioat_probe()
1032 	if (!device->completion_pool) {  in ioat_probe()
1037 	device->enumerate_channels(device);  in ioat_probe()
1047 	err = ioat_dma_setup_interrupts(device);  in ioat_probe()
1051 	err = device->self_test(device);  in ioat_probe()
1058 	ioat_disable_interrupts(device);  in ioat_probe()
1060 	pci_pool_destroy(device->completion_pool);  in ioat_probe()
1062 	pci_pool_destroy(device->dma_pool);  in ioat_probe()
1067 int ioat_register(struct ioatdma_device *device)  in ioat_register()  argument
1069 	int err = dma_async_device_register(&device->common);  in ioat_register()
1072 		ioat_disable_interrupts(device);  in ioat_register()
1073 		pci_pool_destroy(device->completion_pool);  in ioat_register()
1074 		pci_pool_destroy(device->dma_pool);  in ioat_register()
1081 static void ioat1_intr_quirk(struct ioatdma_device *device)  in ioat1_intr_quirk()  argument
1083 	struct pci_dev *pdev = device->pdev;  in ioat1_intr_quirk()
1112 	struct dma_device *dma = c->device;  in cap_show()
1126 	struct dma_device *dma = c->device;  in version_show()
1127 	struct ioatdma_device *device = to_ioatdma_device(dma);  in version_show()  local
1130 		       device->version >> 4, device->version & 0xf);  in version_show()
1165 void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)  in ioat_kobject_add()  argument
1167 	struct dma_device *dma = &device->common;  in ioat_kobject_add()
1172 		struct kobject *parent = &c->dev->device.kobj;  in ioat_kobject_add()
1185 void ioat_kobject_del(struct ioatdma_device *device)  in ioat_kobject_del()  argument
1187 	struct dma_device *dma = &device->common;  in ioat_kobject_del()
1200 int ioat1_dma_probe(struct ioatdma_device *device, int dca)  in ioat1_dma_probe()  argument
1202 	struct pci_dev *pdev = device->pdev;  in ioat1_dma_probe()
1206 	device->intr_quirk = ioat1_intr_quirk;  in ioat1_dma_probe()
1207 	device->enumerate_channels = ioat1_enumerate_channels;  in ioat1_dma_probe()
1208 	device->self_test = ioat_dma_self_test;  in ioat1_dma_probe()
1209 	device->timer_fn = ioat1_timer_event;  in ioat1_dma_probe()
1210 	device->cleanup_fn = ioat1_cleanup_event;  in ioat1_dma_probe()
1211 	dma = &device->common;  in ioat1_dma_probe()
1218 	err = ioat_probe(device);  in ioat1_dma_probe()
1221 	err = ioat_register(device);  in ioat1_dma_probe()
1224 	ioat_kobject_add(device, &ioat1_ktype);  in ioat1_dma_probe()
1227 		device->dca = ioat_dca_init(pdev, device->reg_base);  in ioat1_dma_probe()
1232 void ioat_dma_remove(struct ioatdma_device *device)  in ioat_dma_remove()  argument
1234 	struct dma_device *dma = &device->common;  in ioat_dma_remove()
1236 	ioat_disable_interrupts(device);  in ioat_dma_remove()
1238 	ioat_kobject_del(device);  in ioat_dma_remove()
1242 	pci_pool_destroy(device->dma_pool);  in ioat_dma_remove()
1243 	pci_pool_destroy(device->completion_pool);  in ioat_dma_remove()