ioat_dma          125 drivers/dma/ioat/dma.c 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
ioat_dma          126 drivers/dma/ioat/dma.c 	struct pci_dev *pdev = ioat_dma->pdev;
ioat_dma          136 drivers/dma/ioat/dma.c 	switch (ioat_dma->irq_mode) {
ioat_dma          138 drivers/dma/ioat/dma.c 		msix = &ioat_dma->msix_entries[chan_id];
ioat_dma          363 drivers/dma/ioat/dma.c 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
ioat_dma          431 drivers/dma/ioat/dma.c 	if (ioat_dma->cap & IOAT_CAP_DPS) {
ioat_dma          506 drivers/dma/ioat/dma.c ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed)
ioat_dma          511 drivers/dma/ioat/dma.c 	dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
ioat_dma          578 drivers/dma/ioat/dma.c 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
ioat_dma          606 drivers/dma/ioat/dma.c 		if (ioat_dma->cap & IOAT_CAP_DWBES)
ioat_dma          629 drivers/dma/ioat/dma.c 			ioat_free_sed(ioat_dma, desc->sed);
ioat_dma          651 drivers/dma/ioat/dma.c 		       ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
ioat_dma          708 drivers/dma/ioat/dma.c 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
ioat_dma          747 drivers/dma/ioat/dma.c 			ioat_free_sed(ioat_dma, desc->sed);
ioat_dma          983 drivers/dma/ioat/dma.c 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
ioat_dma          984 drivers/dma/ioat/dma.c 	struct pci_dev *pdev = ioat_dma->pdev;
ioat_dma          994 drivers/dma/ioat/dma.c 	if (ioat_dma->version < IOAT_VER_3_3) {
ioat_dma         1017 drivers/dma/ioat/dma.c 	if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
ioat_dma         1018 drivers/dma/ioat/dma.c 		ioat_dma->msixtba0 = readq(ioat_dma->reg_base + 0x1000);
ioat_dma         1019 drivers/dma/ioat/dma.c 		ioat_dma->msixdata0 = readq(ioat_dma->reg_base + 0x1008);
ioat_dma         1020 drivers/dma/ioat/dma.c 		ioat_dma->msixpba = readq(ioat_dma->reg_base + 0x1800);
ioat_dma         1026 drivers/dma/ioat/dma.c 		if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) {
ioat_dma         1027 drivers/dma/ioat/dma.c 			writeq(ioat_dma->msixtba0, ioat_dma->reg_base + 0x1000);
ioat_dma         1028 drivers/dma/ioat/dma.c 			writeq(ioat_dma->msixdata0, ioat_dma->reg_base + 0x1008);
ioat_dma         1029 drivers/dma/ioat/dma.c 			writeq(ioat_dma->msixpba, ioat_dma->reg_base + 0x1800);
ioat_dma           23 drivers/dma/ioat/dma.h #define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev)
ioat_dma           24 drivers/dma/ioat/dma.h #define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev)
ioat_dma           26 drivers/dma/ioat/dma.h #define chan_num(ch) ((int)((ch)->reg_base - (ch)->ioat_dma->reg_base) / 0x80)
ioat_dma          105 drivers/dma/ioat/dma.h 	struct ioatdma_device *ioat_dma;
ioat_dma          233 drivers/dma/ioat/dma.h ioat_chan_by_index(struct ioatdma_device *ioat_dma, int index)
ioat_dma          235 drivers/dma/ioat/dma.h 	return ioat_dma->idx[index];
ioat_dma          255 drivers/dma/ioat/dma.h 	u8 ver = ioat_chan->ioat_dma->version;
ioat_dma          263 drivers/dma/ioat/dma.h 	u8 ver = ioat_chan->ioat_dma->version;
ioat_dma          271 drivers/dma/ioat/dma.h 	u8 ver = ioat_chan->ioat_dma->version;
ioat_dma          404 drivers/dma/ioat/dma.h void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type);
ioat_dma          405 drivers/dma/ioat/dma.h void ioat_kobject_del(struct ioatdma_device *ioat_dma);
ioat_dma          406 drivers/dma/ioat/dma.h int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma);
ioat_dma          120 drivers/dma/ioat/init.c ioat_init_channel(struct ioatdma_device *ioat_dma,
ioat_dma          122 drivers/dma/ioat/init.c static void ioat_intr_quirk(struct ioatdma_device *ioat_dma);
ioat_dma          123 drivers/dma/ioat/init.c static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma);
ioat_dma          124 drivers/dma/ioat/init.c static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma);
ioat_dma          300 drivers/dma/ioat/init.c static int ioat_dma_self_test(struct ioatdma_device *ioat_dma)
ioat_dma          305 drivers/dma/ioat/init.c 	struct dma_device *dma = &ioat_dma->dma_dev;
ioat_dma          306 drivers/dma/ioat/init.c 	struct device *dev = &ioat_dma->pdev->dev;
ioat_dma          351 drivers/dma/ioat/init.c 	tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest,
ioat_dma          403 drivers/dma/ioat/init.c int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma)
ioat_dma          406 drivers/dma/ioat/init.c 	struct pci_dev *pdev = ioat_dma->pdev;
ioat_dma          424 drivers/dma/ioat/init.c 	msixcnt = ioat_dma->dma_dev.chancnt;
ioat_dma          426 drivers/dma/ioat/init.c 		ioat_dma->msix_entries[i].entry = i;
ioat_dma          428 drivers/dma/ioat/init.c 	err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt);
ioat_dma          433 drivers/dma/ioat/init.c 		msix = &ioat_dma->msix_entries[i];
ioat_dma          434 drivers/dma/ioat/init.c 		ioat_chan = ioat_chan_by_index(ioat_dma, i);
ioat_dma          440 drivers/dma/ioat/init.c 				msix = &ioat_dma->msix_entries[j];
ioat_dma          441 drivers/dma/ioat/init.c 				ioat_chan = ioat_chan_by_index(ioat_dma, j);
ioat_dma          448 drivers/dma/ioat/init.c 	ioat_dma->irq_mode = IOAT_MSIX;
ioat_dma          457 drivers/dma/ioat/init.c 			       "ioat-msi", ioat_dma);
ioat_dma          462 drivers/dma/ioat/init.c 	ioat_dma->irq_mode = IOAT_MSI;
ioat_dma          467 drivers/dma/ioat/init.c 			       IRQF_SHARED, "ioat-intx", ioat_dma);
ioat_dma          471 drivers/dma/ioat/init.c 	ioat_dma->irq_mode = IOAT_INTX;
ioat_dma          474 drivers/dma/ioat/init.c 		ioat_intr_quirk(ioat_dma);
ioat_dma          476 drivers/dma/ioat/init.c 	writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
ioat_dma          481 drivers/dma/ioat/init.c 	writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
ioat_dma          482 drivers/dma/ioat/init.c 	ioat_dma->irq_mode = IOAT_NOIRQ;
ioat_dma          487 drivers/dma/ioat/init.c static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma)
ioat_dma          490 drivers/dma/ioat/init.c 	writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
ioat_dma          493 drivers/dma/ioat/init.c static int ioat_probe(struct ioatdma_device *ioat_dma)
ioat_dma          496 drivers/dma/ioat/init.c 	struct dma_device *dma = &ioat_dma->dma_dev;
ioat_dma          497 drivers/dma/ioat/init.c 	struct pci_dev *pdev = ioat_dma->pdev;
ioat_dma          500 drivers/dma/ioat/init.c 	ioat_dma->completion_pool = dma_pool_create("completion_pool", dev,
ioat_dma          505 drivers/dma/ioat/init.c 	if (!ioat_dma->completion_pool) {
ioat_dma          510 drivers/dma/ioat/init.c 	ioat_enumerate_channels(ioat_dma);
ioat_dma          520 drivers/dma/ioat/init.c 	err = ioat_dma_setup_interrupts(ioat_dma);
ioat_dma          524 drivers/dma/ioat/init.c 	err = ioat3_dma_self_test(ioat_dma);
ioat_dma          531 drivers/dma/ioat/init.c 	ioat_disable_interrupts(ioat_dma);
ioat_dma          533 drivers/dma/ioat/init.c 	dma_pool_destroy(ioat_dma->completion_pool);
ioat_dma          538 drivers/dma/ioat/init.c static int ioat_register(struct ioatdma_device *ioat_dma)
ioat_dma          540 drivers/dma/ioat/init.c 	int err = dma_async_device_register(&ioat_dma->dma_dev);
ioat_dma          543 drivers/dma/ioat/init.c 		ioat_disable_interrupts(ioat_dma);
ioat_dma          544 drivers/dma/ioat/init.c 		dma_pool_destroy(ioat_dma->completion_pool);
ioat_dma          550 drivers/dma/ioat/init.c static void ioat_dma_remove(struct ioatdma_device *ioat_dma)
ioat_dma          552 drivers/dma/ioat/init.c 	struct dma_device *dma = &ioat_dma->dma_dev;
ioat_dma          554 drivers/dma/ioat/init.c 	ioat_disable_interrupts(ioat_dma);
ioat_dma          556 drivers/dma/ioat/init.c 	ioat_kobject_del(ioat_dma);
ioat_dma          560 drivers/dma/ioat/init.c 	dma_pool_destroy(ioat_dma->completion_pool);
ioat_dma          569 drivers/dma/ioat/init.c static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma)
ioat_dma          572 drivers/dma/ioat/init.c 	struct device *dev = &ioat_dma->pdev->dev;
ioat_dma          573 drivers/dma/ioat/init.c 	struct dma_device *dma = &ioat_dma->dma_dev;
ioat_dma          578 drivers/dma/ioat/init.c 	dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET);
ioat_dma          580 drivers/dma/ioat/init.c 	if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) {
ioat_dma          582 drivers/dma/ioat/init.c 			 dma->chancnt, ARRAY_SIZE(ioat_dma->idx));
ioat_dma          583 drivers/dma/ioat/init.c 		dma->chancnt = ARRAY_SIZE(ioat_dma->idx);
ioat_dma          585 drivers/dma/ioat/init.c 	xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET);
ioat_dma          596 drivers/dma/ioat/init.c 		ioat_init_channel(ioat_dma, ioat_chan, i);
ioat_dma          614 drivers/dma/ioat/init.c 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
ioat_dma          630 drivers/dma/ioat/init.c 	if (ioat_dma->version >= IOAT_VER_3_4)
ioat_dma          665 drivers/dma/ioat/init.c 	dma_pool_free(ioat_dma->completion_pool, ioat_chan->completion,
ioat_dma          697 drivers/dma/ioat/init.c 		dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool,
ioat_dma          724 drivers/dma/ioat/init.c 	if (ioat_chan->ioat_dma->version >= IOAT_VER_3_4) {
ioat_dma          766 drivers/dma/ioat/init.c ioat_init_channel(struct ioatdma_device *ioat_dma,
ioat_dma          769 drivers/dma/ioat/init.c 	struct dma_device *dma = &ioat_dma->dma_dev;
ioat_dma          773 drivers/dma/ioat/init.c 	ioat_chan->ioat_dma = ioat_dma;
ioat_dma          774 drivers/dma/ioat/init.c 	ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));
ioat_dma          779 drivers/dma/ioat/init.c 	ioat_dma->idx[idx] = ioat_chan;
ioat_dma          785 drivers/dma/ioat/init.c static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
ioat_dma          802 drivers/dma/ioat/init.c 	struct device *dev = &ioat_dma->pdev->dev;
ioat_dma          803 drivers/dma/ioat/init.c 	struct dma_device *dma = &ioat_dma->dma_dev;
ioat_dma         1050 drivers/dma/ioat/init.c static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma)
ioat_dma         1054 drivers/dma/ioat/init.c 	rc = ioat_dma_self_test(ioat_dma);
ioat_dma         1058 drivers/dma/ioat/init.c 	rc = ioat_xor_val_self_test(ioat_dma);
ioat_dma         1063 drivers/dma/ioat/init.c static void ioat_intr_quirk(struct ioatdma_device *ioat_dma)
ioat_dma         1070 drivers/dma/ioat/init.c 	dma = &ioat_dma->dma_dev;
ioat_dma         1076 drivers/dma/ioat/init.c 	if (ioat_dma->cap & IOAT_CAP_DWBES) {
ioat_dma         1089 drivers/dma/ioat/init.c static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
ioat_dma         1091 drivers/dma/ioat/init.c 	struct pci_dev *pdev = ioat_dma->pdev;
ioat_dma         1099 drivers/dma/ioat/init.c 	dma = &ioat_dma->dma_dev;
ioat_dma         1108 drivers/dma/ioat/init.c 	ioat_dma->cap = readl(ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET);
ioat_dma         1111 drivers/dma/ioat/init.c 		ioat_dma->cap &=
ioat_dma         1115 drivers/dma/ioat/init.c 	if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ)))
ioat_dma         1116 drivers/dma/ioat/init.c 		ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);
ioat_dma         1118 drivers/dma/ioat/init.c 	if (ioat_dma->cap & IOAT_CAP_XOR) {
ioat_dma         1128 drivers/dma/ioat/init.c 	if (ioat_dma->cap & IOAT_CAP_PQ) {
ioat_dma         1135 drivers/dma/ioat/init.c 		if (ioat_dma->cap & IOAT_CAP_RAID16SS)
ioat_dma         1140 drivers/dma/ioat/init.c 		if (!(ioat_dma->cap & IOAT_CAP_XOR)) {
ioat_dma         1146 drivers/dma/ioat/init.c 			if (ioat_dma->cap & IOAT_CAP_RAID16SS)
ioat_dma         1156 drivers/dma/ioat/init.c 	if (ioat_dma->cap & IOAT_CAP_RAID16SS) {
ioat_dma         1164 drivers/dma/ioat/init.c 			ioat_dma->sed_hw_pool[i] = dmam_pool_create(pool_name,
ioat_dma         1167 drivers/dma/ioat/init.c 			if (!ioat_dma->sed_hw_pool[i])
ioat_dma         1173 drivers/dma/ioat/init.c 	if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ)))
ioat_dma         1176 drivers/dma/ioat/init.c 	err = ioat_probe(ioat_dma);
ioat_dma         1186 drivers/dma/ioat/init.c 	err = ioat_register(ioat_dma);
ioat_dma         1190 drivers/dma/ioat/init.c 	ioat_kobject_add(ioat_dma, &ioat_ktype);
ioat_dma         1193 drivers/dma/ioat/init.c 		ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base);
ioat_dma         1206 drivers/dma/ioat/init.c 	if (ioat_dma->cap & IOAT_CAP_DPS)
ioat_dma         1208 drivers/dma/ioat/init.c 		       ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET);
ioat_dma         1215 drivers/dma/ioat/init.c 	struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);
ioat_dma         1219 drivers/dma/ioat/init.c 	if (!ioat_dma)
ioat_dma         1223 drivers/dma/ioat/init.c 		ioat_chan = ioat_dma->idx[i];
ioat_dma         1242 drivers/dma/ioat/init.c 	ioat_disable_interrupts(ioat_dma);
ioat_dma         1245 drivers/dma/ioat/init.c static void ioat_resume(struct ioatdma_device *ioat_dma)
ioat_dma         1252 drivers/dma/ioat/init.c 		ioat_chan = ioat_dma->idx[i];
ioat_dma         1302 drivers/dma/ioat/init.c 	struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev);
ioat_dma         1307 drivers/dma/ioat/init.c 	ioat_resume(ioat_dma);
ioat_dma           80 drivers/dma/ioat/prep.c ioat3_alloc_sed(struct ioatdma_device *ioat_dma, unsigned int hw_pool)
ioat_dma           90 drivers/dma/ioat/prep.c 	sed->hw = dma_pool_alloc(ioat_dma->sed_hw_pool[hw_pool],
ioat_dma          345 drivers/dma/ioat/prep.c 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
ioat_dma          356 drivers/dma/ioat/prep.c 	int cb32 = (ioat_dma->version < IOAT_VER_3_3) ? 1 : 0;
ioat_dma          422 drivers/dma/ioat/prep.c 		if (ioat_dma->cap & IOAT_CAP_DWBES)
ioat_dma          469 drivers/dma/ioat/prep.c 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
ioat_dma          505 drivers/dma/ioat/prep.c 		desc->sed = ioat3_alloc_sed(ioat_dma, (src_cnt-2) >> 3);
ioat_dma          537 drivers/dma/ioat/prep.c 		if (ioat_dma->cap & IOAT_CAP_DWBES)
ioat_dma           34 drivers/dma/ioat/sysfs.c 	struct ioatdma_device *ioat_dma = to_ioatdma_device(dma);
ioat_dma           37 drivers/dma/ioat/sysfs.c 		       ioat_dma->version >> 4, ioat_dma->version & 0xf);
ioat_dma           75 drivers/dma/ioat/sysfs.c void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type)
ioat_dma           77 drivers/dma/ioat/sysfs.c 	struct dma_device *dma = &ioat_dma->dma_dev;
ioat_dma           96 drivers/dma/ioat/sysfs.c void ioat_kobject_del(struct ioatdma_device *ioat_dma)
ioat_dma           98 drivers/dma/ioat/sysfs.c 	struct dma_device *dma = &ioat_dma->dma_dev;