111 	struct at_dma	*atdma = to_at_dma(chan->device);  in atc_alloc_descriptor()  local
114 	desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);  in atc_alloc_descriptor()
227 	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);  in atc_dostart()  local
256 	dma_writel(atdma, CHER, atchan->mask);  in atc_dostart()
451 	struct at_dma			*atdma = to_at_dma(atchan->chan_common.device);  in atc_chain_complete()  local
462 		dma_pool_free(atdma->memset_pool, desc->memset_vaddr,  in atc_chain_complete()
632 	struct at_dma		*atdma = (struct at_dma *)dev_id;  in at_dma_interrupt()  local
639 		imr = dma_readl(atdma, EBCIMR);  in at_dma_interrupt()
640 		status = dma_readl(atdma, EBCISR);  in at_dma_interrupt()
646 		dev_vdbg(atdma->dma_common.dev,  in at_dma_interrupt()
650 		for (i = 0; i < atdma->dma_common.chancnt; i++) {  in at_dma_interrupt()
651 			atchan = &atdma->chan[i];  in at_dma_interrupt()
655 					dma_writel(atdma, CHDR,  in at_dma_interrupt()
936 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_prep_dma_memset()  local
955 	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr);  in atc_prep_dma_memset()
985 	dma_pool_free(atdma->memset_pool, vaddr, paddr);  in atc_prep_dma_memset()
996 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_prep_dma_memset_sg()  local
1013 	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr);  in atc_prep_dma_memset_sg()
1524 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_pause()  local
1534 	dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id));  in atc_pause()
1545 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_resume()  local
1558 	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id));  in atc_resume()
1569 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_terminate_all()  local
1587 	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask);  in atc_terminate_all()
1590 	while (dma_readl(atdma, CHSR) & atchan->mask)  in atc_terminate_all()
1690 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_alloc_chan_resources()  local
1714 		BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);  in atc_alloc_chan_resources()
1730 			dev_err(atdma->dma_common.dev,  in atc_alloc_chan_resources()
1760 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_free_chan_resources()  local
1776 		dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);  in atc_free_chan_resources()
1920 static void at_dma_off(struct at_dma *atdma)  in at_dma_off()  argument
1922 	dma_writel(atdma, EN, 0);  in at_dma_off()
1925 	dma_writel(atdma, EBCIDR, -1L);  in at_dma_off()
1928 	while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)  in at_dma_off()
1935 	struct at_dma		*atdma;  in at_dma_probe()  local
1968 	atdma = kzalloc(size, GFP_KERNEL);  in at_dma_probe()
1969 	if (!atdma)  in at_dma_probe()
1973 	atdma->dma_common.cap_mask = plat_dat->cap_mask;  in at_dma_probe()
1974 	atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1;  in at_dma_probe()
1982 	atdma->regs = ioremap(io->start, size);  in at_dma_probe()
1983 	if (!atdma->regs) {  in at_dma_probe()
1988 	atdma->clk = clk_get(&pdev->dev, "dma_clk");  in at_dma_probe()
1989 	if (IS_ERR(atdma->clk)) {  in at_dma_probe()
1990 		err = PTR_ERR(atdma->clk);  in at_dma_probe()
1993 	err = clk_prepare_enable(atdma->clk);  in at_dma_probe()
1998 	at_dma_off(atdma);  in at_dma_probe()
2000 	err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);  in at_dma_probe()
2004 	platform_set_drvdata(pdev, atdma);  in at_dma_probe()
2007 	atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",  in at_dma_probe()
2010 	if (!atdma->dma_desc_pool) {  in at_dma_probe()
2017 	atdma->memset_pool = dma_pool_create("at_hdmac_memset_pool",  in at_dma_probe()
2019 	if (!atdma->memset_pool) {  in at_dma_probe()
2026 	while (dma_readl(atdma, EBCISR))  in at_dma_probe()
2030 	INIT_LIST_HEAD(&atdma->dma_common.channels);  in at_dma_probe()
2032 		struct at_dma_chan	*atchan = &atdma->chan[i];  in at_dma_probe()
2036 		atchan->chan_common.device = &atdma->dma_common;  in at_dma_probe()
2039 				&atdma->dma_common.channels);  in at_dma_probe()
2041 		atchan->ch_regs = atdma->regs + ch_regs(i);  in at_dma_probe()
2051 		atc_enable_chan_irq(atdma, i);  in at_dma_probe()
2055 	atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;  in at_dma_probe()
2056 	atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;  in at_dma_probe()
2057 	atdma->dma_common.device_tx_status = atc_tx_status;  in at_dma_probe()
2058 	atdma->dma_common.device_issue_pending = atc_issue_pending;  in at_dma_probe()
2059 	atdma->dma_common.dev = &pdev->dev;  in at_dma_probe()
2062 	if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_common.cap_mask))  in at_dma_probe()
2063 		atdma->dma_common.device_prep_interleaved_dma = atc_prep_dma_interleaved;  in at_dma_probe()
2065 	if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))  in at_dma_probe()
2066 		atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;  in at_dma_probe()
2068 	if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) {  in at_dma_probe()
2069 		atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset;  in at_dma_probe()
2070 		atdma->dma_common.device_prep_dma_memset_sg = atc_prep_dma_memset_sg;  in at_dma_probe()
2071 		atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES;  in at_dma_probe()
2074 	if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {  in at_dma_probe()
2075 		atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;  in at_dma_probe()
2077 		dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask);  in at_dma_probe()
2078 		atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic;  in at_dma_probe()
2079 		atdma->dma_common.device_config = atc_config;  in at_dma_probe()
2080 		atdma->dma_common.device_pause = atc_pause;  in at_dma_probe()
2081 		atdma->dma_common.device_resume = atc_resume;  in at_dma_probe()
2082 		atdma->dma_common.device_terminate_all = atc_terminate_all;  in at_dma_probe()
2083 		atdma->dma_common.src_addr_widths = ATC_DMA_BUSWIDTHS;  in at_dma_probe()
2084 		atdma->dma_common.dst_addr_widths = ATC_DMA_BUSWIDTHS;  in at_dma_probe()
2085 		atdma->dma_common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in at_dma_probe()
2086 		atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in at_dma_probe()
2089 	if (dma_has_cap(DMA_SG, atdma->dma_common.cap_mask))  in at_dma_probe()
2090 		atdma->dma_common.device_prep_dma_sg = atc_prep_dma_sg;  in at_dma_probe()
2092 	dma_writel(atdma, EN, AT_DMA_ENABLE);  in at_dma_probe()
2095 	  dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",  in at_dma_probe()
2096 	  dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "",  in at_dma_probe()
2097 	  dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)  ? "slave " : "",  in at_dma_probe()
2098 	  dma_has_cap(DMA_SG, atdma->dma_common.cap_mask)  ? "sg-cpy " : "",  in at_dma_probe()
2101 	dma_async_device_register(&atdma->dma_common);  in at_dma_probe()
2110 						 at_dma_xlate, atdma);  in at_dma_probe()
2120 	dma_async_device_unregister(&atdma->dma_common);  in at_dma_probe()
2121 	dma_pool_destroy(atdma->memset_pool);  in at_dma_probe()
2123 	dma_pool_destroy(atdma->dma_desc_pool);  in at_dma_probe()
2125 	free_irq(platform_get_irq(pdev, 0), atdma);  in at_dma_probe()
2127 	clk_disable_unprepare(atdma->clk);  in at_dma_probe()
2129 	clk_put(atdma->clk);  in at_dma_probe()
2131 	iounmap(atdma->regs);  in at_dma_probe()
2132 	atdma->regs = NULL;  in at_dma_probe()
2136 	kfree(atdma);  in at_dma_probe()
2142 	struct at_dma		*atdma = platform_get_drvdata(pdev);  in at_dma_remove()  local
2146 	at_dma_off(atdma);  in at_dma_remove()
2147 	dma_async_device_unregister(&atdma->dma_common);  in at_dma_remove()
2149 	dma_pool_destroy(atdma->memset_pool);  in at_dma_remove()
2150 	dma_pool_destroy(atdma->dma_desc_pool);  in at_dma_remove()
2151 	free_irq(platform_get_irq(pdev, 0), atdma);  in at_dma_remove()
2153 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_remove()
2158 		atc_disable_chan_irq(atdma, chan->chan_id);  in at_dma_remove()
2164 	clk_disable_unprepare(atdma->clk);  in at_dma_remove()
2165 	clk_put(atdma->clk);  in at_dma_remove()
2167 	iounmap(atdma->regs);  in at_dma_remove()
2168 	atdma->regs = NULL;  in at_dma_remove()
2173 	kfree(atdma);  in at_dma_remove()
2180 	struct at_dma	*atdma = platform_get_drvdata(pdev);  in at_dma_shutdown()  local
2183 	clk_disable_unprepare(atdma->clk);  in at_dma_shutdown()
2189 	struct at_dma *atdma = platform_get_drvdata(pdev);  in at_dma_prepare()  local
2192 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_prepare()
2224 	struct at_dma *atdma = platform_get_drvdata(pdev);  in at_dma_suspend_noirq()  local
2228 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_suspend_noirq()
2236 	atdma->save_imr = dma_readl(atdma, EBCIMR);  in at_dma_suspend_noirq()
2239 	at_dma_off(atdma);  in at_dma_suspend_noirq()
2240 	clk_disable_unprepare(atdma->clk);  in at_dma_suspend_noirq()
2246 	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);  in atc_resume_cyclic()  local
2255 	dma_writel(atdma, CHER, atchan->mask);  in atc_resume_cyclic()
2266 	struct at_dma *atdma = platform_get_drvdata(pdev);  in at_dma_resume_noirq()  local
2270 	clk_prepare_enable(atdma->clk);  in at_dma_resume_noirq()
2271 	dma_writel(atdma, EN, AT_DMA_ENABLE);  in at_dma_resume_noirq()
2274 	while (dma_readl(atdma, EBCISR))  in at_dma_resume_noirq()
2278 	dma_writel(atdma, EBCIER, atdma->save_imr);  in at_dma_resume_noirq()
2279 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_resume_noirq()