109 	struct at_dma	*atdma = to_at_dma(chan->device);  in atc_alloc_descriptor()  local
112 	desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);  in atc_alloc_descriptor()
225 	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);  in atc_dostart()  local
250 	dma_writel(atdma, CHER, atchan->mask);  in atc_dostart()
562 	struct at_dma		*atdma = (struct at_dma *)dev_id;  in at_dma_interrupt()  local
569 		imr = dma_readl(atdma, EBCIMR);  in at_dma_interrupt()
570 		status = dma_readl(atdma, EBCISR);  in at_dma_interrupt()
576 		dev_vdbg(atdma->dma_common.dev,  in at_dma_interrupt()
580 		for (i = 0; i < atdma->dma_common.chancnt; i++) {  in at_dma_interrupt()
581 			atchan = &atdma->chan[i];  in at_dma_interrupt()
585 					dma_writel(atdma, CHDR,  in at_dma_interrupt()
1191 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_pause()  local
1201 	dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id));  in atc_pause()
1212 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_resume()  local
1225 	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id));  in atc_resume()
1236 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_terminate_all()  local
1254 	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask);  in atc_terminate_all()
1257 	while (dma_readl(atdma, CHSR) & atchan->mask)  in atc_terminate_all()
1357 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_alloc_chan_resources()  local
1381 		BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);  in atc_alloc_chan_resources()
1397 			dev_err(atdma->dma_common.dev,  in atc_alloc_chan_resources()
1427 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_free_chan_resources()  local
1443 		dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);  in atc_free_chan_resources()
1587 static void at_dma_off(struct at_dma *atdma)  in at_dma_off()  argument
1589 	dma_writel(atdma, EN, 0);  in at_dma_off()
1592 	dma_writel(atdma, EBCIDR, -1L);  in at_dma_off()
1595 	while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)  in at_dma_off()
1602 	struct at_dma		*atdma;  in at_dma_probe()  local
1631 	atdma = kzalloc(size, GFP_KERNEL);  in at_dma_probe()
1632 	if (!atdma)  in at_dma_probe()
1636 	atdma->dma_common.cap_mask = plat_dat->cap_mask;  in at_dma_probe()
1637 	atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1;  in at_dma_probe()
1645 	atdma->regs = ioremap(io->start, size);  in at_dma_probe()
1646 	if (!atdma->regs) {  in at_dma_probe()
1651 	atdma->clk = clk_get(&pdev->dev, "dma_clk");  in at_dma_probe()
1652 	if (IS_ERR(atdma->clk)) {  in at_dma_probe()
1653 		err = PTR_ERR(atdma->clk);  in at_dma_probe()
1656 	err = clk_prepare_enable(atdma->clk);  in at_dma_probe()
1661 	at_dma_off(atdma);  in at_dma_probe()
1663 	err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);  in at_dma_probe()
1667 	platform_set_drvdata(pdev, atdma);  in at_dma_probe()
1670 	atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",  in at_dma_probe()
1673 	if (!atdma->dma_desc_pool) {  in at_dma_probe()
1680 	while (dma_readl(atdma, EBCISR))  in at_dma_probe()
1684 	INIT_LIST_HEAD(&atdma->dma_common.channels);  in at_dma_probe()
1686 		struct at_dma_chan	*atchan = &atdma->chan[i];  in at_dma_probe()
1690 		atchan->chan_common.device = &atdma->dma_common;  in at_dma_probe()
1693 				&atdma->dma_common.channels);  in at_dma_probe()
1695 		atchan->ch_regs = atdma->regs + ch_regs(i);  in at_dma_probe()
1705 		atc_enable_chan_irq(atdma, i);  in at_dma_probe()
1709 	atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;  in at_dma_probe()
1710 	atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;  in at_dma_probe()
1711 	atdma->dma_common.device_tx_status = atc_tx_status;  in at_dma_probe()
1712 	atdma->dma_common.device_issue_pending = atc_issue_pending;  in at_dma_probe()
1713 	atdma->dma_common.dev = &pdev->dev;  in at_dma_probe()
1716 	if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))  in at_dma_probe()
1717 		atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;  in at_dma_probe()
1719 	if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {  in at_dma_probe()
1720 		atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;  in at_dma_probe()
1722 		dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask);  in at_dma_probe()
1723 		atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic;  in at_dma_probe()
1724 		atdma->dma_common.device_config = atc_config;  in at_dma_probe()
1725 		atdma->dma_common.device_pause = atc_pause;  in at_dma_probe()
1726 		atdma->dma_common.device_resume = atc_resume;  in at_dma_probe()
1727 		atdma->dma_common.device_terminate_all = atc_terminate_all;  in at_dma_probe()
1728 		atdma->dma_common.src_addr_widths = ATC_DMA_BUSWIDTHS;  in at_dma_probe()
1729 		atdma->dma_common.dst_addr_widths = ATC_DMA_BUSWIDTHS;  in at_dma_probe()
1730 		atdma->dma_common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in at_dma_probe()
1731 		atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in at_dma_probe()
1734 	if (dma_has_cap(DMA_SG, atdma->dma_common.cap_mask))  in at_dma_probe()
1735 		atdma->dma_common.device_prep_dma_sg = atc_prep_dma_sg;  in at_dma_probe()
1737 	dma_writel(atdma, EN, AT_DMA_ENABLE);  in at_dma_probe()
1740 	  dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",  in at_dma_probe()
1741 	  dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)  ? "slave " : "",  in at_dma_probe()
1742 	  dma_has_cap(DMA_SG, atdma->dma_common.cap_mask)  ? "sg-cpy " : "",  in at_dma_probe()
1745 	dma_async_device_register(&atdma->dma_common);  in at_dma_probe()
1754 						 at_dma_xlate, atdma);  in at_dma_probe()
1764 	dma_async_device_unregister(&atdma->dma_common);  in at_dma_probe()
1765 	dma_pool_destroy(atdma->dma_desc_pool);  in at_dma_probe()
1767 	free_irq(platform_get_irq(pdev, 0), atdma);  in at_dma_probe()
1769 	clk_disable_unprepare(atdma->clk);  in at_dma_probe()
1771 	clk_put(atdma->clk);  in at_dma_probe()
1773 	iounmap(atdma->regs);  in at_dma_probe()
1774 	atdma->regs = NULL;  in at_dma_probe()
1778 	kfree(atdma);  in at_dma_probe()
1784 	struct at_dma		*atdma = platform_get_drvdata(pdev);  in at_dma_remove()  local
1788 	at_dma_off(atdma);  in at_dma_remove()
1789 	dma_async_device_unregister(&atdma->dma_common);  in at_dma_remove()
1791 	dma_pool_destroy(atdma->dma_desc_pool);  in at_dma_remove()
1792 	free_irq(platform_get_irq(pdev, 0), atdma);  in at_dma_remove()
1794 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_remove()
1799 		atc_disable_chan_irq(atdma, chan->chan_id);  in at_dma_remove()
1805 	clk_disable_unprepare(atdma->clk);  in at_dma_remove()
1806 	clk_put(atdma->clk);  in at_dma_remove()
1808 	iounmap(atdma->regs);  in at_dma_remove()
1809 	atdma->regs = NULL;  in at_dma_remove()
1814 	kfree(atdma);  in at_dma_remove()
1821 	struct at_dma	*atdma = platform_get_drvdata(pdev);  in at_dma_shutdown()  local
1824 	clk_disable_unprepare(atdma->clk);  in at_dma_shutdown()
1830 	struct at_dma *atdma = platform_get_drvdata(pdev);  in at_dma_prepare()  local
1833 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_prepare()
1865 	struct at_dma *atdma = platform_get_drvdata(pdev);  in at_dma_suspend_noirq()  local
1869 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_suspend_noirq()
1877 	atdma->save_imr = dma_readl(atdma, EBCIMR);  in at_dma_suspend_noirq()
1880 	at_dma_off(atdma);  in at_dma_suspend_noirq()
1881 	clk_disable_unprepare(atdma->clk);  in at_dma_suspend_noirq()
1887 	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);  in atc_resume_cyclic()  local
1896 	dma_writel(atdma, CHER, atchan->mask);  in atc_resume_cyclic()
1907 	struct at_dma *atdma = platform_get_drvdata(pdev);  in at_dma_resume_noirq()  local
1911 	clk_prepare_enable(atdma->clk);  in at_dma_resume_noirq()
1912 	dma_writel(atdma, EN, AT_DMA_ENABLE);  in at_dma_resume_noirq()
1915 	while (dma_readl(atdma, EBCISR))  in at_dma_resume_noirq()
1919 	dma_writel(atdma, EBCIER, atdma->save_imr);  in at_dma_resume_noirq()
1920 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_resume_noirq()