195 	struct dma_chan			chan;  member
226 	struct at_xdmac_chan	chan[0];  member
272 	return container_of(dchan, struct at_xdmac_chan, chan);  in to_at_xdmac_chan()
275 static struct device *chan2dev(struct dma_chan *chan)  in chan2dev()  argument
277 	return &chan->dev->device;  in chan2dev()
342 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()
345 	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);  in at_xdmac_start_xfer()
382 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
404 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
409 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
423 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(tx->chan);  in at_xdmac_tx_submit()
430 	dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n",  in at_xdmac_tx_submit()
440 static struct at_xdmac_desc *at_xdmac_alloc_desc(struct dma_chan *chan,  in at_xdmac_alloc_desc()  argument
444 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_alloc_desc()
451 		dma_async_tx_descriptor_init(&desc->tx_dma_desc, chan);  in at_xdmac_alloc_desc()
474 		desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT);  in at_xdmac_get_desc()
485 static void at_xdmac_queue_desc(struct dma_chan *chan,  in at_xdmac_queue_desc()  argument
495 	dev_dbg(chan2dev(chan),	"%s: chain lld: prev=0x%p, mbr_nda=%pad\n",  in at_xdmac_queue_desc()
499 static inline void at_xdmac_increment_block_count(struct dma_chan *chan,  in at_xdmac_increment_block_count()  argument
507 	dev_dbg(chan2dev(chan),  in at_xdmac_increment_block_count()
517 	struct dma_chan		*chan;  in at_xdmac_xlate()  local
525 	chan = dma_get_any_slave_channel(&atxdmac->dma);  in at_xdmac_xlate()
526 	if (!chan) {  in at_xdmac_xlate()
531 	atchan = to_at_xdmac_chan(chan);  in at_xdmac_xlate()
538 	return chan;  in at_xdmac_xlate()
541 static int at_xdmac_compute_chan_conf(struct dma_chan *chan,  in at_xdmac_compute_chan_conf()  argument
544 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_compute_chan_conf()
560 			dev_err(chan2dev(chan), "invalid src maxburst value\n");  in at_xdmac_compute_chan_conf()
566 			dev_err(chan2dev(chan), "invalid src addr width value\n");  in at_xdmac_compute_chan_conf()
583 			dev_err(chan2dev(chan), "invalid src maxburst value\n");  in at_xdmac_compute_chan_conf()
589 			dev_err(chan2dev(chan), "invalid dst addr width value\n");  in at_xdmac_compute_chan_conf()
595 	dev_dbg(chan2dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);  in at_xdmac_compute_chan_conf()
618 static int at_xdmac_set_slave_config(struct dma_chan *chan,  in at_xdmac_set_slave_config()  argument
621 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_set_slave_config()
624 		dev_err(chan2dev(chan), "invalid slave configuration\n");  in at_xdmac_set_slave_config()
634 at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,  in at_xdmac_prep_slave_sg()  argument
638 	struct at_xdmac_chan		*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_slave_sg()
650 		dev_err(chan2dev(chan), "invalid DMA direction\n");  in at_xdmac_prep_slave_sg()
654 	dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n",  in at_xdmac_prep_slave_sg()
662 	if (at_xdmac_compute_chan_conf(chan, direction))  in at_xdmac_prep_slave_sg()
673 			dev_err(chan2dev(chan), "sg data length is zero\n");  in at_xdmac_prep_slave_sg()
676 		dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n",  in at_xdmac_prep_slave_sg()
681 			dev_err(chan2dev(chan), "can't get descriptor\n");  in at_xdmac_prep_slave_sg()
705 		dev_dbg(chan2dev(chan),  in at_xdmac_prep_slave_sg()
711 			at_xdmac_queue_desc(chan, prev, desc);  in at_xdmac_prep_slave_sg()
717 		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",  in at_xdmac_prep_slave_sg()
735 at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,  in at_xdmac_prep_dma_cyclic()  argument
740 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_cyclic()
746 	dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n",  in at_xdmac_prep_dma_cyclic()
751 		dev_err(chan2dev(chan), "invalid DMA direction\n");  in at_xdmac_prep_dma_cyclic()
756 		dev_err(chan2dev(chan), "channel currently used\n");  in at_xdmac_prep_dma_cyclic()
760 	if (at_xdmac_compute_chan_conf(chan, direction))  in at_xdmac_prep_dma_cyclic()
769 			dev_err(chan2dev(chan), "can't get descriptor\n");  in at_xdmac_prep_dma_cyclic()
776 		dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_cyclic()
793 		dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_cyclic()
799 			at_xdmac_queue_desc(chan, prev, desc);  in at_xdmac_prep_dma_cyclic()
805 		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",  in at_xdmac_prep_dma_cyclic()
810 	at_xdmac_queue_desc(chan, prev, first);  in at_xdmac_prep_dma_cyclic()
818 static inline u32 at_xdmac_align_width(struct dma_chan *chan, dma_addr_t addr)  in at_xdmac_align_width()  argument
832 		dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__);  in at_xdmac_align_width()
835 		dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__);  in at_xdmac_align_width()
838 		dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__);  in at_xdmac_align_width()
841 		dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__);  in at_xdmac_align_width()
848 at_xdmac_interleaved_queue_desc(struct dma_chan *chan,  in at_xdmac_interleaved_queue_desc()  argument
873 	dwidth = at_xdmac_align_width(chan, src | dst | chunk->size);  in at_xdmac_interleaved_queue_desc()
875 		dev_dbg(chan2dev(chan),  in at_xdmac_interleaved_queue_desc()
883 		dev_dbg(chan2dev(chan),  in at_xdmac_interleaved_queue_desc()
904 		dev_err(chan2dev(chan), "can't get descriptor\n");  in at_xdmac_interleaved_queue_desc()
923 	dev_dbg(chan2dev(chan),  in at_xdmac_interleaved_queue_desc()
930 		at_xdmac_queue_desc(chan, prev, desc);  in at_xdmac_interleaved_queue_desc()
936 at_xdmac_prep_interleaved(struct dma_chan *chan,  in at_xdmac_prep_interleaved()  argument
940 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_interleaved()
957 	dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, numf=%d, frame_size=%d, flags=0x%lx\n",  in at_xdmac_prep_interleaved()
965 		first = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
972 			at_xdmac_increment_block_count(chan, first);  in at_xdmac_prep_interleaved()
974 		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",  in at_xdmac_prep_interleaved()
990 			dev_dbg(chan2dev(chan),  in at_xdmac_prep_interleaved()
994 			desc = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
1007 			dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",  in at_xdmac_prep_interleaved()
1030 at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,  in at_xdmac_prep_dma_memcpy()  argument
1033 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memcpy()
1054 	dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n",  in at_xdmac_prep_dma_memcpy()
1060 	dwidth = at_xdmac_align_width(chan, src_addr | dst_addr);  in at_xdmac_prep_dma_memcpy()
1066 		dev_dbg(chan2dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size);  in at_xdmac_prep_dma_memcpy()
1072 			dev_err(chan2dev(chan), "can't get descriptor\n");  in at_xdmac_prep_dma_memcpy()
1087 		dev_dbg(chan2dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size);  in at_xdmac_prep_dma_memcpy()
1090 		dwidth = at_xdmac_align_width(chan,  in at_xdmac_prep_dma_memcpy()
1106 		dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_memcpy()
1112 			at_xdmac_queue_desc(chan, prev, desc);  in at_xdmac_prep_dma_memcpy()
1118 		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",  in at_xdmac_prep_dma_memcpy()
1129 static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,  in at_xdmac_memset_create_desc()  argument
1156 	dwidth = at_xdmac_align_width(chan, dst_addr);  in at_xdmac_memset_create_desc()
1159 		dev_err(chan2dev(chan),  in at_xdmac_memset_create_desc()
1169 		dev_err(chan2dev(chan), "can't get descriptor\n");  in at_xdmac_memset_create_desc()
1185 	dev_dbg(chan2dev(chan),  in at_xdmac_memset_create_desc()
1194 at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,  in at_xdmac_prep_dma_memset()  argument
1197 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset()
1200 	dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n",  in at_xdmac_prep_dma_memset()
1206 	desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value);  in at_xdmac_prep_dma_memset()
1217 at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl,  in at_xdmac_prep_dma_memset_sg()  argument
1221 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset_sg()
1231 	dev_dbg(chan2dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n",  in at_xdmac_prep_dma_memset_sg()
1236 		dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n",  in at_xdmac_prep_dma_memset_sg()
1239 		desc = at_xdmac_memset_create_desc(chan, atchan,  in at_xdmac_prep_dma_memset_sg()
1278 				dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_memset_sg()
1286 				at_xdmac_increment_block_count(chan, ppdesc);  in at_xdmac_prep_dma_memset_sg()
1317 				at_xdmac_queue_desc(chan, ppdesc, pdesc);  in at_xdmac_prep_dma_memset_sg()
1326 				dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_memset_sg()
1341 			dev_dbg(chan2dev(chan),  in at_xdmac_prep_dma_memset_sg()
1349 			at_xdmac_increment_block_count(chan, pdesc);  in at_xdmac_prep_dma_memset_sg()
1379 at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,  in at_xdmac_tx_status()  argument
1382 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_tx_status()
1383 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()
1392 	ret = dma_cookie_status(chan, cookie, txstate);  in at_xdmac_tx_status()
1477 	dev_dbg(chan2dev(chan),  in at_xdmac_tx_status()
1490 	dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_remove_xfer()
1515 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_advance_work()
1541 	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08lx\n",  in at_xdmac_tasklet()
1555 			dev_err(chan2dev(&atchan->chan), "read bus error!!!");  in at_xdmac_tasklet()
1557 			dev_err(chan2dev(&atchan->chan), "write bus error!!!");  in at_xdmac_tasklet()
1559 			dev_err(chan2dev(&atchan->chan), "request overflow error!!!");  in at_xdmac_tasklet()
1565 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_tasklet()
1610 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1617 			dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_interrupt()
1639 static void at_xdmac_issue_pending(struct dma_chan *chan)  in at_xdmac_issue_pending()  argument
1641 	struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);  in at_xdmac_issue_pending()
1643 	dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);  in at_xdmac_issue_pending()
1651 static int at_xdmac_device_config(struct dma_chan *chan,  in at_xdmac_device_config()  argument
1654 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_config()
1658 	dev_dbg(chan2dev(chan), "%s\n", __func__);  in at_xdmac_device_config()
1661 	ret = at_xdmac_set_slave_config(chan, config);  in at_xdmac_device_config()
1667 static int at_xdmac_device_pause(struct dma_chan *chan)  in at_xdmac_device_pause()  argument
1669 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_pause()
1670 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()
1673 	dev_dbg(chan2dev(chan), "%s\n", __func__);  in at_xdmac_device_pause()
1688 static int at_xdmac_device_resume(struct dma_chan *chan)  in at_xdmac_device_resume()  argument
1690 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_resume()
1691 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()
1694 	dev_dbg(chan2dev(chan), "%s\n", __func__);  in at_xdmac_device_resume()
1709 static int at_xdmac_device_terminate_all(struct dma_chan *chan)  in at_xdmac_device_terminate_all()  argument
1712 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_terminate_all()
1713 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()
1716 	dev_dbg(chan2dev(chan), "%s\n", __func__);  in at_xdmac_device_terminate_all()
1734 static int at_xdmac_alloc_chan_resources(struct dma_chan *chan)  in at_xdmac_alloc_chan_resources()  argument
1736 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_alloc_chan_resources()
1744 		dev_err(chan2dev(chan),  in at_xdmac_alloc_chan_resources()
1751 		dev_err(chan2dev(chan),  in at_xdmac_alloc_chan_resources()
1758 		desc = at_xdmac_alloc_desc(chan, GFP_ATOMIC);  in at_xdmac_alloc_chan_resources()
1760 			dev_warn(chan2dev(chan),  in at_xdmac_alloc_chan_resources()
1767 	dma_cookie_init(chan);  in at_xdmac_alloc_chan_resources()
1769 	dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);  in at_xdmac_alloc_chan_resources()
1776 static void at_xdmac_free_chan_resources(struct dma_chan *chan)  in at_xdmac_free_chan_resources()  argument
1778 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_free_chan_resources()
1779 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_free_chan_resources()
1783 		dev_dbg(chan2dev(chan), "%s: freeing descriptor %p\n", __func__, desc);  in at_xdmac_free_chan_resources()
1796 	struct dma_chan		*chan, *_chan;  in atmel_xdmac_prepare()  local
1798 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_prepare()
1799 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_prepare()
1816 	struct dma_chan		*chan, *_chan;  in atmel_xdmac_suspend()  local
1818 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_suspend()
1819 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_suspend()
1824 				at_xdmac_device_pause(chan);  in atmel_xdmac_suspend()
1842 	struct dma_chan		*chan, *_chan;  in atmel_xdmac_resume()  local
1849 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1856 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_resume()
1857 		atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_resume()
1861 				at_xdmac_device_resume(chan);  in atmel_xdmac_resume()
1982 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()
1984 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
1985 		list_add_tail(&atchan->chan.device_node,  in at_xdmac_probe()
2045 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()
2048 		at_xdmac_free_chan_resources(&atchan->chan);  in at_xdmac_remove()