73 static struct device *chan2dev(struct dma_chan *chan)  in chan2dev()  argument
75 	return &chan->dev->device;  in chan2dev()
98 		dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);  in dwc_desc_get()
102 	dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);  in dwc_desc_get()
120 			dev_vdbg(chan2dev(&dwc->chan),  in dwc_desc_put()
124 		dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);  in dwc_desc_put()
132 	struct dw_dma *dw = to_dw_dma(dwc->chan.device);  in dwc_initialize()
171 	dev_err(chan2dev(&dwc->chan),  in dwc_dump_chan_regs()
193 	struct dw_dma	*dw = to_dw_dma(dwc->chan.device);  in dwc_do_single_block()
215 	struct dw_dma	*dw = to_dw_dma(dwc->chan.device);  in dwc_dostart()
220 		dev_err(chan2dev(&dwc->chan),  in dwc_dostart()
233 			dev_err(chan2dev(&dwc->chan),  in dwc_dostart()
267 	dev_vdbg(chan2dev(&dwc->chan), "%s: started %u\n", __func__, desc->txd.cookie);  in dwc_dostart_first_queued()
283 	dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);  in dwc_descriptor_complete()
315 		dev_err(chan2dev(&dwc->chan),  in dwc_complete_all()
405 		dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__);  in dwc_scan_descriptors()
410 	dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp);  in dwc_scan_descriptors()
450 	dev_err(chan2dev(&dwc->chan),  in dwc_scan_descriptors()
462 	dev_crit(chan2dev(&dwc->chan), "  desc: s0x%x d0x%x l0x%x c0x%x:%x\n",  in dwc_dump_lli()
497 	dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"  in dwc_handle_error()
511 dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan)  in dw_dma_get_src_addr()  argument
513 	struct dw_dma_chan *dwc = to_dw_dma_chan(chan);  in dw_dma_get_src_addr()
518 dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan)  in dw_dma_get_dst_addr()  argument
520 	struct dw_dma_chan *dwc = to_dw_dma_chan(chan);  in dw_dma_get_dst_addr()
535 		dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",  in dwc_handle_cyclic()
554 		dev_err(chan2dev(&dwc->chan),  in dwc_handle_cyclic()
601 		dwc = &dw->chan[i];  in dw_dma_tasklet()
659 	struct dw_dma_chan	*dwc = to_dw_dma_chan(tx->chan);  in dwc_tx_submit()
672 	dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", __func__, desc->txd.cookie);  in dwc_tx_submit()
681 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,  in dwc_prep_dma_memcpy()  argument
684 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dwc_prep_dma_memcpy()
685 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_prep_dma_memcpy()
696 	dev_vdbg(chan2dev(chan),  in dwc_prep_dma_memcpy()
701 		dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);  in dwc_prep_dma_memcpy()
713 	ctllo = DWC_DEFAULT_CTLLO(chan)  in dwc_prep_dma_memcpy()
761 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,  in dwc_prep_slave_sg()  argument
765 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dwc_prep_slave_sg()
766 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_prep_slave_sg()
779 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in dwc_prep_slave_sg()
792 		ctllo = (DWC_DEFAULT_CTLLO(chan)  in dwc_prep_slave_sg()
849 		ctllo = (DWC_DEFAULT_CTLLO(chan)  in dwc_prep_slave_sg()
916 	dev_err(chan2dev(chan),  in dwc_prep_slave_sg()
922 bool dw_dma_filter(struct dma_chan *chan, void *param)  in dw_dma_filter()  argument
924 	struct dw_dma_chan *dwc = to_dw_dma_chan(chan);  in dw_dma_filter()
927 	if (dws->dma_dev != chan->device->dev)  in dw_dma_filter()
958 static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig)  in dwc_config()  argument
960 	struct dw_dma_chan *dwc = to_dw_dma_chan(chan);  in dwc_config()
975 static int dwc_pause(struct dma_chan *chan)  in dwc_pause()  argument
977 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dwc_pause()
1005 static int dwc_resume(struct dma_chan *chan)  in dwc_resume()  argument
1007 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dwc_resume()
1022 static int dwc_terminate_all(struct dma_chan *chan)  in dwc_terminate_all()  argument
1024 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dwc_terminate_all()
1025 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_terminate_all()
1067 dwc_tx_status(struct dma_chan *chan,  in dwc_tx_status()  argument
1071 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dwc_tx_status()
1074 	ret = dma_cookie_status(chan, cookie, txstate);  in dwc_tx_status()
1078 	dwc_scan_descriptors(to_dw_dma(chan->device), dwc);  in dwc_tx_status()
1080 	ret = dma_cookie_status(chan, cookie, txstate);  in dwc_tx_status()
1090 static void dwc_issue_pending(struct dma_chan *chan)  in dwc_issue_pending()  argument
1092 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dwc_issue_pending()
1119 		dw->chan[i].initialized = false;  in dw_dma_off()
1127 static int dwc_alloc_chan_resources(struct dma_chan *chan)  in dwc_alloc_chan_resources()  argument
1129 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dwc_alloc_chan_resources()
1130 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_alloc_chan_resources()
1135 	dev_vdbg(chan2dev(chan), "%s\n", __func__);  in dwc_alloc_chan_resources()
1139 		dev_dbg(chan2dev(chan), "DMA channel not idle?\n");  in dwc_alloc_chan_resources()
1143 	dma_cookie_init(chan);  in dwc_alloc_chan_resources()
1154 	if (chan->private && !dw_dma_filter(chan, chan->private)) {  in dwc_alloc_chan_resources()
1155 		dev_warn(chan2dev(chan), "Wrong controller-specific data\n");  in dwc_alloc_chan_resources()
1178 		dma_async_tx_descriptor_init(&desc->txd, chan);  in dwc_alloc_chan_resources()
1191 	dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);  in dwc_alloc_chan_resources()
1196 	dev_info(chan2dev(chan), "only allocated %d descriptors\n", i);  in dwc_alloc_chan_resources()
1201 static void dwc_free_chan_resources(struct dma_chan *chan)  in dwc_free_chan_resources()  argument
1203 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dwc_free_chan_resources()
1204 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_free_chan_resources()
1209 	dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,  in dwc_free_chan_resources()
1215 	BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);  in dwc_free_chan_resources()
1243 		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);  in dwc_free_chan_resources()
1247 	dev_vdbg(chan2dev(chan), "%s: done\n", __func__);  in dwc_free_chan_resources()
1259 int dw_dma_cyclic_start(struct dma_chan *chan)  in dw_dma_cyclic_start()  argument
1261 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dw_dma_cyclic_start()
1262 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dw_dma_cyclic_start()
1266 		dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n");  in dw_dma_cyclic_start()
1289 void dw_dma_cyclic_stop(struct dma_chan *chan)  in dw_dma_cyclic_stop()  argument
1291 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dw_dma_cyclic_stop()
1292 	struct dw_dma		*dw = to_dw_dma(dwc->chan.device);  in dw_dma_cyclic_stop()
1314 struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,  in dw_dma_cyclic_prep()  argument
1318 	struct dw_dma_chan		*dwc = to_dw_dma_chan(chan);  in dw_dma_cyclic_prep()
1333 		dev_dbg(chan2dev(&dwc->chan),  in dw_dma_cyclic_prep()
1340 		dev_dbg(chan2dev(&dwc->chan),  in dw_dma_cyclic_prep()
1348 		dev_dbg(chan2dev(&dwc->chan),  in dw_dma_cyclic_prep()
1397 			desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)  in dw_dma_cyclic_prep()
1412 			desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)  in dw_dma_cyclic_prep()
1440 	dev_dbg(chan2dev(&dwc->chan),  in dw_dma_cyclic_prep()
1464 void dw_dma_cyclic_free(struct dma_chan *chan)  in dw_dma_cyclic_free()  argument
1466 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);  in dw_dma_cyclic_free()
1467 	struct dw_dma		*dw = to_dw_dma(dwc->chan.device);  in dw_dma_cyclic_free()
1472 	dev_dbg(chan2dev(&dwc->chan), "%s\n", __func__);  in dw_dma_cyclic_free()
1552 	dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan),  in dw_dma_probe()
1554 	if (!dw->chan) {  in dw_dma_probe()
1588 		struct dw_dma_chan	*dwc = &dw->chan[i];  in dw_dma_probe()
1590 		dwc->chan.device = &dw->dma;  in dw_dma_probe()
1591 		dma_cookie_init(&dwc->chan);  in dw_dma_probe()
1593 			list_add_tail(&dwc->chan.device_node,  in dw_dma_probe()
1596 			list_add(&dwc->chan.device_node, &dw->dma.channels);  in dw_dma_probe()
1716 			chan.device_node) {  in dw_dma_remove()
1717 		list_del(&dwc->chan.device_node);  in dw_dma_remove()