ioat_dma 125 drivers/dma/ioat/dma.c struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; ioat_dma 126 drivers/dma/ioat/dma.c struct pci_dev *pdev = ioat_dma->pdev; ioat_dma 136 drivers/dma/ioat/dma.c switch (ioat_dma->irq_mode) { ioat_dma 138 drivers/dma/ioat/dma.c msix = &ioat_dma->msix_entries[chan_id]; ioat_dma 363 drivers/dma/ioat/dma.c struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; ioat_dma 431 drivers/dma/ioat/dma.c if (ioat_dma->cap & IOAT_CAP_DPS) { ioat_dma 506 drivers/dma/ioat/dma.c ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed) ioat_dma 511 drivers/dma/ioat/dma.c dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma); ioat_dma 578 drivers/dma/ioat/dma.c struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; ioat_dma 606 drivers/dma/ioat/dma.c if (ioat_dma->cap & IOAT_CAP_DWBES) ioat_dma 629 drivers/dma/ioat/dma.c ioat_free_sed(ioat_dma, desc->sed); ioat_dma 651 drivers/dma/ioat/dma.c ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET); ioat_dma 708 drivers/dma/ioat/dma.c struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; ioat_dma 747 drivers/dma/ioat/dma.c ioat_free_sed(ioat_dma, desc->sed); ioat_dma 983 drivers/dma/ioat/dma.c struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; ioat_dma 984 drivers/dma/ioat/dma.c struct pci_dev *pdev = ioat_dma->pdev; ioat_dma 994 drivers/dma/ioat/dma.c if (ioat_dma->version < IOAT_VER_3_3) { ioat_dma 1017 drivers/dma/ioat/dma.c if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) { ioat_dma 1018 drivers/dma/ioat/dma.c ioat_dma->msixtba0 = readq(ioat_dma->reg_base + 0x1000); ioat_dma 1019 drivers/dma/ioat/dma.c ioat_dma->msixdata0 = readq(ioat_dma->reg_base + 0x1008); ioat_dma 1020 drivers/dma/ioat/dma.c ioat_dma->msixpba = readq(ioat_dma->reg_base + 0x1800); ioat_dma 1026 drivers/dma/ioat/dma.c if (is_bwd_ioat(pdev) && (ioat_dma->irq_mode == IOAT_MSIX)) { ioat_dma 1027 drivers/dma/ioat/dma.c writeq(ioat_dma->msixtba0, ioat_dma->reg_base + 0x1000); ioat_dma 1028 drivers/dma/ioat/dma.c writeq(ioat_dma->msixdata0, ioat_dma->reg_base + 0x1008); ioat_dma 1029 drivers/dma/ioat/dma.c writeq(ioat_dma->msixpba, ioat_dma->reg_base + 0x1800); ioat_dma 23 drivers/dma/ioat/dma.h #define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev) ioat_dma 24 drivers/dma/ioat/dma.h #define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev) ioat_dma 26 drivers/dma/ioat/dma.h #define chan_num(ch) ((int)((ch)->reg_base - (ch)->ioat_dma->reg_base) / 0x80) ioat_dma 105 drivers/dma/ioat/dma.h struct ioatdma_device *ioat_dma; ioat_dma 233 drivers/dma/ioat/dma.h ioat_chan_by_index(struct ioatdma_device *ioat_dma, int index) ioat_dma 235 drivers/dma/ioat/dma.h return ioat_dma->idx[index]; ioat_dma 255 drivers/dma/ioat/dma.h u8 ver = ioat_chan->ioat_dma->version; ioat_dma 263 drivers/dma/ioat/dma.h u8 ver = ioat_chan->ioat_dma->version; ioat_dma 271 drivers/dma/ioat/dma.h u8 ver = ioat_chan->ioat_dma->version; ioat_dma 404 drivers/dma/ioat/dma.h void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type); ioat_dma 405 drivers/dma/ioat/dma.h void ioat_kobject_del(struct ioatdma_device *ioat_dma); ioat_dma 406 drivers/dma/ioat/dma.h int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma); ioat_dma 120 drivers/dma/ioat/init.c ioat_init_channel(struct ioatdma_device *ioat_dma, ioat_dma 122 drivers/dma/ioat/init.c static void ioat_intr_quirk(struct ioatdma_device *ioat_dma); ioat_dma 123 drivers/dma/ioat/init.c static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma); ioat_dma 124 drivers/dma/ioat/init.c static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma); ioat_dma 300 drivers/dma/ioat/init.c static int ioat_dma_self_test(struct ioatdma_device *ioat_dma) ioat_dma 305 drivers/dma/ioat/init.c struct dma_device *dma = &ioat_dma->dma_dev; ioat_dma 306 drivers/dma/ioat/init.c struct device *dev = &ioat_dma->pdev->dev; ioat_dma 351 drivers/dma/ioat/init.c tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest, ioat_dma 403 drivers/dma/ioat/init.c int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma) ioat_dma 406 drivers/dma/ioat/init.c struct pci_dev *pdev = ioat_dma->pdev; ioat_dma 424 drivers/dma/ioat/init.c msixcnt = ioat_dma->dma_dev.chancnt; ioat_dma 426 drivers/dma/ioat/init.c ioat_dma->msix_entries[i].entry = i; ioat_dma 428 drivers/dma/ioat/init.c err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt); ioat_dma 433 drivers/dma/ioat/init.c msix = &ioat_dma->msix_entries[i]; ioat_dma 434 drivers/dma/ioat/init.c ioat_chan = ioat_chan_by_index(ioat_dma, i); ioat_dma 440 drivers/dma/ioat/init.c msix = &ioat_dma->msix_entries[j]; ioat_dma 441 drivers/dma/ioat/init.c ioat_chan = ioat_chan_by_index(ioat_dma, j); ioat_dma 448 drivers/dma/ioat/init.c ioat_dma->irq_mode = IOAT_MSIX; ioat_dma 457 drivers/dma/ioat/init.c "ioat-msi", ioat_dma); ioat_dma 462 drivers/dma/ioat/init.c ioat_dma->irq_mode = IOAT_MSI; ioat_dma 467 drivers/dma/ioat/init.c IRQF_SHARED, "ioat-intx", ioat_dma); ioat_dma 471 drivers/dma/ioat/init.c ioat_dma->irq_mode = IOAT_INTX; ioat_dma 474 drivers/dma/ioat/init.c ioat_intr_quirk(ioat_dma); ioat_dma 476 drivers/dma/ioat/init.c writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); ioat_dma 481 drivers/dma/ioat/init.c writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); ioat_dma 482 drivers/dma/ioat/init.c ioat_dma->irq_mode = IOAT_NOIRQ; ioat_dma 487 drivers/dma/ioat/init.c static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma) ioat_dma 490 drivers/dma/ioat/init.c writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); ioat_dma 493 drivers/dma/ioat/init.c static int ioat_probe(struct ioatdma_device *ioat_dma) ioat_dma 496 drivers/dma/ioat/init.c struct dma_device *dma = &ioat_dma->dma_dev; ioat_dma 497 drivers/dma/ioat/init.c struct pci_dev *pdev = ioat_dma->pdev; ioat_dma 500 drivers/dma/ioat/init.c ioat_dma->completion_pool = dma_pool_create("completion_pool", dev, ioat_dma 505 drivers/dma/ioat/init.c if (!ioat_dma->completion_pool) { ioat_dma 510 drivers/dma/ioat/init.c ioat_enumerate_channels(ioat_dma); ioat_dma 520 drivers/dma/ioat/init.c err = ioat_dma_setup_interrupts(ioat_dma); ioat_dma 524 drivers/dma/ioat/init.c err = ioat3_dma_self_test(ioat_dma); ioat_dma 531 drivers/dma/ioat/init.c ioat_disable_interrupts(ioat_dma); ioat_dma 533 drivers/dma/ioat/init.c dma_pool_destroy(ioat_dma->completion_pool); ioat_dma 538 drivers/dma/ioat/init.c static int ioat_register(struct ioatdma_device *ioat_dma) ioat_dma 540 drivers/dma/ioat/init.c int err = dma_async_device_register(&ioat_dma->dma_dev); ioat_dma 543 drivers/dma/ioat/init.c ioat_disable_interrupts(ioat_dma); ioat_dma 544 drivers/dma/ioat/init.c dma_pool_destroy(ioat_dma->completion_pool); ioat_dma 550 drivers/dma/ioat/init.c static void ioat_dma_remove(struct ioatdma_device *ioat_dma) ioat_dma 552 drivers/dma/ioat/init.c struct dma_device *dma = &ioat_dma->dma_dev; ioat_dma 554 drivers/dma/ioat/init.c ioat_disable_interrupts(ioat_dma); ioat_dma 556 drivers/dma/ioat/init.c ioat_kobject_del(ioat_dma); ioat_dma 560 drivers/dma/ioat/init.c dma_pool_destroy(ioat_dma->completion_pool); ioat_dma 569 drivers/dma/ioat/init.c static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma) ioat_dma 572 drivers/dma/ioat/init.c struct device *dev = &ioat_dma->pdev->dev; ioat_dma 573 drivers/dma/ioat/init.c struct dma_device *dma = &ioat_dma->dma_dev; ioat_dma 578 drivers/dma/ioat/init.c dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET); ioat_dma 580 drivers/dma/ioat/init.c if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) { ioat_dma 582 drivers/dma/ioat/init.c dma->chancnt, ARRAY_SIZE(ioat_dma->idx)); ioat_dma 583 drivers/dma/ioat/init.c dma->chancnt = ARRAY_SIZE(ioat_dma->idx); ioat_dma 585 drivers/dma/ioat/init.c xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET); ioat_dma 596 drivers/dma/ioat/init.c ioat_init_channel(ioat_dma, ioat_chan, i); ioat_dma 614 drivers/dma/ioat/init.c struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; ioat_dma 630 drivers/dma/ioat/init.c if (ioat_dma->version >= IOAT_VER_3_4) ioat_dma 665 drivers/dma/ioat/init.c dma_pool_free(ioat_dma->completion_pool, ioat_chan->completion, ioat_dma 697 drivers/dma/ioat/init.c dma_pool_zalloc(ioat_chan->ioat_dma->completion_pool, ioat_dma 724 drivers/dma/ioat/init.c if (ioat_chan->ioat_dma->version >= IOAT_VER_3_4) { ioat_dma 766 drivers/dma/ioat/init.c ioat_init_channel(struct ioatdma_device *ioat_dma, ioat_dma 769 drivers/dma/ioat/init.c struct dma_device *dma = &ioat_dma->dma_dev; ioat_dma 773 drivers/dma/ioat/init.c ioat_chan->ioat_dma = ioat_dma; ioat_dma 774 drivers/dma/ioat/init.c ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1)); ioat_dma 779 drivers/dma/ioat/init.c ioat_dma->idx[idx] = ioat_chan; ioat_dma 785 drivers/dma/ioat/init.c static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) ioat_dma 802 drivers/dma/ioat/init.c struct device *dev = &ioat_dma->pdev->dev; ioat_dma 803 drivers/dma/ioat/init.c struct dma_device *dma = &ioat_dma->dma_dev; ioat_dma 1050 drivers/dma/ioat/init.c static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma) ioat_dma 1054 drivers/dma/ioat/init.c rc = ioat_dma_self_test(ioat_dma); ioat_dma 1058 drivers/dma/ioat/init.c rc = ioat_xor_val_self_test(ioat_dma); ioat_dma 1063 drivers/dma/ioat/init.c static void ioat_intr_quirk(struct ioatdma_device *ioat_dma) ioat_dma 1070 drivers/dma/ioat/init.c dma = &ioat_dma->dma_dev; ioat_dma 1076 drivers/dma/ioat/init.c if (ioat_dma->cap & IOAT_CAP_DWBES) { ioat_dma 1089 drivers/dma/ioat/init.c static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) ioat_dma 1091 drivers/dma/ioat/init.c struct pci_dev *pdev = ioat_dma->pdev; ioat_dma 1099 drivers/dma/ioat/init.c dma = &ioat_dma->dma_dev; ioat_dma 1108 drivers/dma/ioat/init.c ioat_dma->cap = readl(ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET); ioat_dma 1111 drivers/dma/ioat/init.c ioat_dma->cap &= ioat_dma 1115 drivers/dma/ioat/init.c if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ))) ioat_dma 1116 drivers/dma/ioat/init.c ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ); ioat_dma 1118 drivers/dma/ioat/init.c if (ioat_dma->cap & IOAT_CAP_XOR) { ioat_dma 1128 drivers/dma/ioat/init.c if (ioat_dma->cap & IOAT_CAP_PQ) { ioat_dma 1135 drivers/dma/ioat/init.c if (ioat_dma->cap & IOAT_CAP_RAID16SS) ioat_dma 1140 drivers/dma/ioat/init.c if (!(ioat_dma->cap & IOAT_CAP_XOR)) { ioat_dma 1146 drivers/dma/ioat/init.c if (ioat_dma->cap & IOAT_CAP_RAID16SS) ioat_dma 1156 drivers/dma/ioat/init.c if (ioat_dma->cap & IOAT_CAP_RAID16SS) { ioat_dma 1164 drivers/dma/ioat/init.c ioat_dma->sed_hw_pool[i] = dmam_pool_create(pool_name, ioat_dma 1167 drivers/dma/ioat/init.c if (!ioat_dma->sed_hw_pool[i]) ioat_dma 1173 drivers/dma/ioat/init.c if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ))) ioat_dma 1176 drivers/dma/ioat/init.c err = ioat_probe(ioat_dma); ioat_dma 1186 drivers/dma/ioat/init.c err = ioat_register(ioat_dma); ioat_dma 1190 drivers/dma/ioat/init.c ioat_kobject_add(ioat_dma, &ioat_ktype); ioat_dma 1193 drivers/dma/ioat/init.c ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base); ioat_dma 1206 drivers/dma/ioat/init.c if (ioat_dma->cap & IOAT_CAP_DPS) ioat_dma 1208 drivers/dma/ioat/init.c ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET); ioat_dma 1215 drivers/dma/ioat/init.c struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev); ioat_dma 1219 drivers/dma/ioat/init.c if (!ioat_dma) ioat_dma 1223 drivers/dma/ioat/init.c ioat_chan = ioat_dma->idx[i]; ioat_dma 1242 drivers/dma/ioat/init.c ioat_disable_interrupts(ioat_dma); ioat_dma 1245 drivers/dma/ioat/init.c static void ioat_resume(struct ioatdma_device *ioat_dma) ioat_dma 1252 drivers/dma/ioat/init.c ioat_chan = ioat_dma->idx[i]; ioat_dma 1302 drivers/dma/ioat/init.c struct ioatdma_device *ioat_dma = pci_get_drvdata(pdev); ioat_dma 1307 drivers/dma/ioat/init.c ioat_resume(ioat_dma); ioat_dma 80 drivers/dma/ioat/prep.c ioat3_alloc_sed(struct ioatdma_device *ioat_dma, unsigned int hw_pool) ioat_dma 90 drivers/dma/ioat/prep.c sed->hw = dma_pool_alloc(ioat_dma->sed_hw_pool[hw_pool], ioat_dma 345 drivers/dma/ioat/prep.c struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; ioat_dma 356 drivers/dma/ioat/prep.c int cb32 = (ioat_dma->version < IOAT_VER_3_3) ? 1 : 0; ioat_dma 422 drivers/dma/ioat/prep.c if (ioat_dma->cap & IOAT_CAP_DWBES) ioat_dma 469 drivers/dma/ioat/prep.c struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; ioat_dma 505 drivers/dma/ioat/prep.c desc->sed = ioat3_alloc_sed(ioat_dma, (src_cnt-2) >> 3); ioat_dma 537 drivers/dma/ioat/prep.c if (ioat_dma->cap & IOAT_CAP_DWBES) ioat_dma 34 drivers/dma/ioat/sysfs.c struct ioatdma_device *ioat_dma = to_ioatdma_device(dma); ioat_dma 37 drivers/dma/ioat/sysfs.c ioat_dma->version >> 4, ioat_dma->version & 0xf); ioat_dma 75 drivers/dma/ioat/sysfs.c void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type) ioat_dma 77 drivers/dma/ioat/sysfs.c struct dma_device *dma = &ioat_dma->dma_dev; ioat_dma 96 drivers/dma/ioat/sysfs.c void ioat_kobject_del(struct ioatdma_device *ioat_dma) ioat_dma 98 drivers/dma/ioat/sysfs.c struct dma_device *dma = &ioat_dma->dma_dev;