65 static void atc_issue_pending(struct dma_chan *chan);
107 static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,  in atc_alloc_descriptor()  argument
111 	struct at_dma	*atdma = to_at_dma(chan->device);  in atc_alloc_descriptor()
118 		dma_async_tx_descriptor_init(&desc->txd, chan);  in atc_alloc_descriptor()
310 static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie)  in atc_get_bytes_left()  argument
312 	struct at_dma_chan      *atchan = to_at_dma_chan(chan);  in atc_get_bytes_left()
651 			atchan = &atdma->chan[i];  in at_dma_interrupt()
684 	struct at_dma_chan	*atchan = to_at_dma_chan(tx->chan);  in atc_tx_submit()
692 		dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",  in atc_tx_submit()
697 		dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",  in atc_tx_submit()
714 atc_prep_dma_interleaved(struct dma_chan *chan,  in atc_prep_dma_interleaved()  argument
718 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_dma_interleaved()
731 	dev_info(chan2dev(chan),  in atc_prep_dma_interleaved()
748 			dev_err(chan2dev(chan),  in atc_prep_dma_interleaved()
762 		dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__);  in atc_prep_dma_interleaved()
779 		dev_err(chan2dev(chan),  in atc_prep_dma_interleaved()
813 atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,  in atc_prep_dma_memcpy()  argument
816 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_dma_memcpy()
827 	dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n",  in atc_prep_dma_memcpy()
831 		dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");  in atc_prep_dma_memcpy()
884 static struct at_desc *atc_create_memset_desc(struct dma_chan *chan,  in atc_create_memset_desc()  argument
889 	struct at_dma_chan *atchan = to_at_dma_chan(chan);  in atc_create_memset_desc()
901 		dev_err(chan2dev(chan), "%s: buffer is too big\n",  in atc_create_memset_desc()
908 		dev_err(chan2dev(chan), "%s: can't get a descriptor\n",  in atc_create_memset_desc()
933 atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,  in atc_prep_dma_memset()  argument
936 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_prep_dma_memset()
941 	dev_vdbg(chan2dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__,  in atc_prep_dma_memset()
945 		dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);  in atc_prep_dma_memset()
949 	if (!is_dma_fill_aligned(chan->device, dest, 0, len)) {  in atc_prep_dma_memset()
950 		dev_dbg(chan2dev(chan), "%s: buffer is not aligned\n",  in atc_prep_dma_memset()
957 		dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n",  in atc_prep_dma_memset()
963 	desc = atc_create_memset_desc(chan, paddr, dest, len);  in atc_prep_dma_memset()
965 		dev_err(chan2dev(chan), "%s: couldn't get a descriptor\n",  in atc_prep_dma_memset()
990 atc_prep_dma_memset_sg(struct dma_chan *chan,  in atc_prep_dma_memset_sg()  argument
995 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_dma_memset_sg()
996 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_prep_dma_memset_sg()
1004 	dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%zx f0x%lx\n", __func__,  in atc_prep_dma_memset_sg()
1008 		dev_dbg(chan2dev(chan), "%s: scatterlist is empty!\n",  in atc_prep_dma_memset_sg()
1015 		dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n",  in atc_prep_dma_memset_sg()
1025 		dev_vdbg(chan2dev(chan), "%s: d%pad, l0x%zx\n",  in atc_prep_dma_memset_sg()
1028 		if (!is_dma_fill_aligned(chan->device, dest, 0, len)) {  in atc_prep_dma_memset_sg()
1029 			dev_err(chan2dev(chan), "%s: buffer is not aligned\n",  in atc_prep_dma_memset_sg()
1034 		desc = atc_create_memset_desc(chan, paddr, dest, len);  in atc_prep_dma_memset_sg()
1076 atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,  in atc_prep_slave_sg()  argument
1080 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_slave_sg()
1081 	struct at_dma_slave	*atslave = chan->private;  in atc_prep_slave_sg()
1094 	dev_vdbg(chan2dev(chan), "prep_slave_sg (%d): %s f0x%lx\n",  in atc_prep_slave_sg()
1100 		dev_dbg(chan2dev(chan), "prep_slave_sg: sg length is zero!\n");  in atc_prep_slave_sg()
1129 				dev_dbg(chan2dev(chan),  in atc_prep_slave_sg()
1170 				dev_dbg(chan2dev(chan),  in atc_prep_slave_sg()
1207 	dev_err(chan2dev(chan), "not enough descriptors available\n");  in atc_prep_slave_sg()
1223 atc_prep_dma_sg(struct dma_chan *chan,  in atc_prep_dma_sg()  argument
1228 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_dma_sg()
1370 atc_dma_cyclic_fill_desc(struct dma_chan *chan, struct at_desc *desc,  in atc_dma_cyclic_fill_desc()  argument
1375 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_dma_cyclic_fill_desc()
1428 atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,  in atc_prep_dma_cyclic()  argument
1432 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_prep_dma_cyclic()
1433 	struct at_dma_slave	*atslave = chan->private;  in atc_prep_dma_cyclic()
1442 	dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%d/%d)\n",  in atc_prep_dma_cyclic()
1448 		dev_dbg(chan2dev(chan), "prep_dma_cyclic: length is zero!\n");  in atc_prep_dma_cyclic()
1454 		dev_dbg(chan2dev(chan), "prep_dma_cyclic: channel in use!\n");  in atc_prep_dma_cyclic()
1478 		if (atc_dma_cyclic_fill_desc(chan, desc, i, buf_addr,  in atc_prep_dma_cyclic()
1495 	dev_err(chan2dev(chan), "not enough descriptors available\n");  in atc_prep_dma_cyclic()
1502 static int atc_config(struct dma_chan *chan,  in atc_config()  argument
1505 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_config()
1507 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in atc_config()
1510 	if (!chan->private)  in atc_config()
1521 static int atc_pause(struct dma_chan *chan)  in atc_pause()  argument
1523 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_pause()
1524 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_pause()
1530 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in atc_pause()
1542 static int atc_resume(struct dma_chan *chan)  in atc_resume()  argument
1544 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_resume()
1545 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_resume()
1551 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in atc_resume()
1566 static int atc_terminate_all(struct dma_chan *chan)  in atc_terminate_all()  argument
1568 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_terminate_all()
1569 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_terminate_all()
1576 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in atc_terminate_all()
1621 atc_tx_status(struct dma_chan *chan,  in atc_tx_status()  argument
1625 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_tx_status()
1630 	ret = dma_cookie_status(chan, cookie, txstate);  in atc_tx_status()
1643 	bytes = atc_get_bytes_left(chan, cookie);  in atc_tx_status()
1648 		dev_vdbg(chan2dev(chan), "get residual bytes error\n");  in atc_tx_status()
1654 	dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %d\n",  in atc_tx_status()
1664 static void atc_issue_pending(struct dma_chan *chan)  in atc_issue_pending()  argument
1666 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_issue_pending()
1669 	dev_vdbg(chan2dev(chan), "issue_pending\n");  in atc_issue_pending()
1687 static int atc_alloc_chan_resources(struct dma_chan *chan)  in atc_alloc_chan_resources()  argument
1689 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_alloc_chan_resources()
1690 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_alloc_chan_resources()
1698 	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");  in atc_alloc_chan_resources()
1702 		dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");  in atc_alloc_chan_resources()
1708 	atslave = chan->private;  in atc_alloc_chan_resources()
1728 		desc = atc_alloc_descriptor(chan, GFP_KERNEL);  in atc_alloc_chan_resources()
1740 	dma_cookie_init(chan);  in atc_alloc_chan_resources()
1746 	dev_dbg(chan2dev(chan),  in atc_alloc_chan_resources()
1757 static void atc_free_chan_resources(struct dma_chan *chan)  in atc_free_chan_resources()  argument
1759 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in atc_free_chan_resources()
1760 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_free_chan_resources()
1764 	dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n",  in atc_free_chan_resources()
1773 		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);  in atc_free_chan_resources()
1782 	dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");  in atc_free_chan_resources()
1786 static bool at_dma_filter(struct dma_chan *chan, void *slave)  in at_dma_filter()  argument
1790 	if (atslave->dma_dev == chan->device->dev) {  in at_dma_filter()
1791 		chan->private = atslave;  in at_dma_filter()
1801 	struct dma_chan *chan;  in at_dma_xlate()  local
1846 	chan = dma_request_channel(mask, at_dma_filter, atslave);  in at_dma_xlate()
1847 	if (!chan)  in at_dma_xlate()
1850 	atchan = to_at_dma_chan(chan);  in at_dma_xlate()
1854 	return chan;  in at_dma_xlate()
2032 		struct at_dma_chan	*atchan = &atdma->chan[i];  in at_dma_probe()
2143 	struct dma_chan		*chan, *_chan;  in at_dma_remove()  local
2153 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_remove()
2155 		struct at_dma_chan	*atchan = to_at_dma_chan(chan);  in at_dma_remove()
2158 		atc_disable_chan_irq(atdma, chan->chan_id);  in at_dma_remove()
2161 		list_del(&chan->device_node);  in at_dma_remove()
2190 	struct dma_chan *chan, *_chan;  in at_dma_prepare()  local
2192 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_prepare()
2194 		struct at_dma_chan *atchan = to_at_dma_chan(chan);  in at_dma_prepare()
2204 	struct dma_chan	*chan = &atchan->chan_common;  in atc_suspend_cyclic()  local
2209 		dev_warn(chan2dev(chan),  in atc_suspend_cyclic()
2211 		atc_pause(chan);  in atc_suspend_cyclic()
2225 	struct dma_chan *chan, *_chan;  in at_dma_suspend_noirq()  local
2228 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_suspend_noirq()
2230 		struct at_dma_chan *atchan = to_at_dma_chan(chan);  in at_dma_suspend_noirq()
2267 	struct dma_chan *chan, *_chan;  in at_dma_resume_noirq()  local
2279 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_resume_noirq()
2281 		struct at_dma_chan *atchan = to_at_dma_chan(chan);  in at_dma_resume_noirq()