139 	container_of(dchan, struct pxad_chan, vc.chan)
151 		dev_vdbg(&phy->vchan->vc.chan.dev->device,		\
159 		dev_vdbg(&phy->vchan->vc.chan.dev->device,		\
166 		dev_vdbg(&phy->vchan->vc.chan.dev->device,		\
336 	struct dentry *chan, *chan_state = NULL, *chan_descr = NULL;  in pxad_dbg_alloc_chan()  local
341 	chan = debugfs_create_dir(chan_name, chandir);  in pxad_dbg_alloc_chan()
344 	if (chan)  in pxad_dbg_alloc_chan()
345 		chan_state = debugfs_create_file("state", 0400, chan, dt,  in pxad_dbg_alloc_chan()
348 		chan_descr = debugfs_create_file("descriptors", 0400, chan, dt,  in pxad_dbg_alloc_chan()
351 		chan_reqs = debugfs_create_file("requesters", 0400, chan, dt,  in pxad_dbg_alloc_chan()
356 	return chan;  in pxad_dbg_alloc_chan()
359 	debugfs_remove_recursive(chan);  in pxad_dbg_alloc_chan()
425 	struct pxad_device *pdev = to_pxad_dev(pchan->vc.chan.device);  in lookup_phy()
457 	dev_dbg(&pchan->vc.chan.dev->device,  in lookup_phy()
464 static void pxad_free_phy(struct pxad_chan *chan)  in pxad_free_phy()  argument
466 	struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device);  in pxad_free_phy()
471 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_free_phy()
473 	if (!chan->phy)  in pxad_free_phy()
477 	if (chan->drcmr <= pdev->nr_requestors) {  in pxad_free_phy()
478 		reg = pxad_drcmr(chan->drcmr);  in pxad_free_phy()
479 		writel_relaxed(0, chan->phy->base + reg);  in pxad_free_phy()
484 		if (chan->phy == &pdev->phys[i])  in pxad_free_phy()
486 	chan->phy->vchan = NULL;  in pxad_free_phy()
487 	chan->phy = NULL;  in pxad_free_phy()
491 static bool is_chan_running(struct pxad_chan *chan)  in is_chan_running()  argument
494 	struct pxad_phy *phy = chan->phy;  in is_chan_running()
502 static bool is_running_chan_misaligned(struct pxad_chan *chan)  in is_running_chan_misaligned()  argument
506 	BUG_ON(!chan->phy);  in is_running_chan_misaligned()
507 	dalgn = phy_readl_relaxed(chan->phy, DALGN);  in is_running_chan_misaligned()
508 	return dalgn & (BIT(chan->phy->idx));  in is_running_chan_misaligned()
519 	dev_dbg(&phy->vchan->vc.chan.dev->device,  in phy_enable()
523 	pdev = to_pxad_dev(phy->vchan->vc.chan.device);  in phy_enable()
548 	dev_dbg(&phy->vchan->vc.chan.dev->device,  in phy_disable()
553 static void pxad_launch_chan(struct pxad_chan *chan,  in pxad_launch_chan()  argument
556 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_launch_chan()
558 	if (!chan->phy) {  in pxad_launch_chan()
559 		chan->phy = lookup_phy(chan);  in pxad_launch_chan()
560 		if (!chan->phy) {  in pxad_launch_chan()
561 			dev_dbg(&chan->vc.chan.dev->device,  in pxad_launch_chan()
571 	phy_writel(chan->phy, desc->first, DDADR);  in pxad_launch_chan()
572 	phy_enable(chan->phy, chan->misaligned);  in pxad_launch_chan()
617 	struct pxad_chan *chan = to_pxad_chan(&vc->chan);  in pxad_try_hotchain()  local
626 	if (is_chan_running(chan)) {  in pxad_try_hotchain()
629 		if (!is_running_chan_misaligned(chan) &&  in pxad_try_hotchain()
636 		if (is_chan_running(chan) || is_desc_completed(vd_last_issued))  in pxad_try_hotchain()
655 		dev_warn(&phy->vchan->vc.chan.dev->device,  in clear_chan_irq()
665 	struct pxad_chan *chan = phy->vchan;  in pxad_chan_handler()  local
670 	BUG_ON(!chan);  in pxad_chan_handler()
676 	spin_lock_irqsave(&chan->vc.lock, flags);  in pxad_chan_handler()
677 	list_for_each_entry_safe(vd, tmp, &chan->vc.desc_issued, node) {  in pxad_chan_handler()
678 		dev_dbg(&chan->vc.chan.dev->device,  in pxad_chan_handler()
694 		dev_dbg(&chan->vc.chan.dev->device,  in pxad_chan_handler()
697 			list_empty(&chan->vc.desc_submitted),  in pxad_chan_handler()
698 			list_empty(&chan->vc.desc_issued));  in pxad_chan_handler()
701 		if (list_empty(&chan->vc.desc_issued)) {  in pxad_chan_handler()
702 			chan->misaligned =  in pxad_chan_handler()
703 				!list_empty(&chan->vc.desc_submitted);  in pxad_chan_handler()
705 			vd = list_first_entry(&chan->vc.desc_issued,  in pxad_chan_handler()
707 			pxad_launch_chan(chan, to_pxad_sw_desc(vd));  in pxad_chan_handler()
710 	spin_unlock_irqrestore(&chan->vc.lock, flags);  in pxad_chan_handler()
737 	struct pxad_chan *chan = to_pxad_chan(dchan);  in pxad_alloc_chan_resources()  local
738 	struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device);  in pxad_alloc_chan_resources()
740 	if (chan->desc_pool)  in pxad_alloc_chan_resources()
743 	chan->desc_pool = dma_pool_create(dma_chan_name(dchan),  in pxad_alloc_chan_resources()
748 	if (!chan->desc_pool) {  in pxad_alloc_chan_resources()
749 		dev_err(&chan->vc.chan.dev->device,  in pxad_alloc_chan_resources()
760 	struct pxad_chan *chan = to_pxad_chan(dchan);  in pxad_free_chan_resources()  local
762 	vchan_free_chan_resources(&chan->vc);  in pxad_free_chan_resources()
763 	dma_pool_destroy(chan->desc_pool);  in pxad_free_chan_resources()
764 	chan->desc_pool = NULL;  in pxad_free_chan_resources()
788 pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc)  in pxad_alloc_desc()  argument
799 	sw_desc->desc_pool = chan->desc_pool;  in pxad_alloc_desc()
805 			dev_err(&chan->vc.chan.dev->device,  in pxad_alloc_desc()
826 	struct virt_dma_chan *vc = to_virt_chan(tx->chan);  in pxad_tx_submit()
827 	struct pxad_chan *chan = to_pxad_chan(&vc->chan);  in pxad_tx_submit()  local
840 		dev_dbg(&chan->vc.chan.dev->device,  in pxad_tx_submit()
858 		if (chan->misaligned || !to_pxad_sw_desc(vd)->misaligned)  in pxad_tx_submit()
863 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_tx_submit()
867 	chan->misaligned |= to_pxad_sw_desc(vd)->misaligned;  in pxad_tx_submit()
876 	struct pxad_chan *chan = to_pxad_chan(dchan);  in pxad_issue_pending()  local
880 	spin_lock_irqsave(&chan->vc.lock, flags);  in pxad_issue_pending()
881 	if (list_empty(&chan->vc.desc_submitted))  in pxad_issue_pending()
884 	vd_first = list_first_entry(&chan->vc.desc_submitted,  in pxad_issue_pending()
886 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_issue_pending()
889 	vchan_issue_pending(&chan->vc);  in pxad_issue_pending()
890 	if (!pxad_try_hotchain(&chan->vc, vd_first))  in pxad_issue_pending()
891 		pxad_launch_chan(chan, to_pxad_sw_desc(vd_first));  in pxad_issue_pending()
893 	spin_unlock_irqrestore(&chan->vc.lock, flags);  in pxad_issue_pending()
901 	struct pxad_chan *chan = container_of(vc, struct pxad_chan, vc);  in pxad_tx_prep()  local
906 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_tx_prep()
914 static void pxad_get_config(struct pxad_chan *chan,  in pxad_get_config()  argument
920 	struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device);  in pxad_get_config()
924 		maxburst = chan->cfg.src_maxburst;  in pxad_get_config()
925 		width = chan->cfg.src_addr_width;  in pxad_get_config()
926 		dev_addr = chan->cfg.src_addr;  in pxad_get_config()
929 		if (chan->drcmr <= pdev->nr_requestors)  in pxad_get_config()
933 		maxburst = chan->cfg.dst_maxburst;  in pxad_get_config()
934 		width = chan->cfg.dst_addr_width;  in pxad_get_config()
935 		dev_addr = chan->cfg.dst_addr;  in pxad_get_config()
938 		if (chan->drcmr <= pdev->nr_requestors)  in pxad_get_config()
945 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_get_config()
967 	if (chan->cfg.slave_id)  in pxad_get_config()
968 		chan->drcmr = chan->cfg.slave_id;  in pxad_get_config()
976 	struct pxad_chan *chan = to_pxad_chan(dchan);  in pxad_prep_memcpy()  local
986 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_prep_memcpy()
990 	pxad_get_config(chan, DMA_MEM_TO_MEM, &dcmd, NULL, NULL);  in pxad_prep_memcpy()
993 	sw_desc = pxad_alloc_desc(chan, nb_desc + 1);  in pxad_prep_memcpy()
1015 	return pxad_tx_prep(&chan->vc, &sw_desc->vd, flags);  in pxad_prep_memcpy()
1023 	struct pxad_chan *chan = to_pxad_chan(dchan);  in pxad_prep_slave_sg()  local
1034 	pxad_get_config(chan, dir, &dcmd, &dsadr, &dtadr);  in pxad_prep_slave_sg()
1035 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_prep_slave_sg()
1040 	sw_desc = pxad_alloc_desc(chan, nb_desc + 1);  in pxad_prep_slave_sg()
1065 	return pxad_tx_prep(&chan->vc, &sw_desc->vd, flags);  in pxad_prep_slave_sg()
1073 	struct pxad_chan *chan = to_pxad_chan(dchan);  in pxad_prep_dma_cyclic()  local
1083 		dev_err(&chan->vc.chan.dev->device,  in pxad_prep_dma_cyclic()
1092 	pxad_get_config(chan, dir, &dcmd, &dsadr, &dtadr);  in pxad_prep_dma_cyclic()
1094 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_prep_dma_cyclic()
1100 	sw_desc = pxad_alloc_desc(chan, nb_desc + 1);  in pxad_prep_dma_cyclic()
1118 	return pxad_tx_prep(&chan->vc, &sw_desc->vd, flags);  in pxad_prep_dma_cyclic()
1124 	struct pxad_chan *chan = to_pxad_chan(dchan);  in pxad_config()  local
1129 	chan->cfg = *cfg;  in pxad_config()
1135 	struct pxad_chan *chan = to_pxad_chan(dchan);  in pxad_terminate_all()  local
1136 	struct pxad_device *pdev = to_pxad_dev(chan->vc.chan.device);  in pxad_terminate_all()
1142 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_terminate_all()
1143 		"%s(): vchan %p: terminate all\n", __func__, &chan->vc);  in pxad_terminate_all()
1145 	spin_lock_irqsave(&chan->vc.lock, flags);  in pxad_terminate_all()
1146 	vchan_get_all_descriptors(&chan->vc, &head);  in pxad_terminate_all()
1149 		dev_dbg(&chan->vc.chan.dev->device,  in pxad_terminate_all()
1154 	phy = chan->phy;  in pxad_terminate_all()
1156 		phy_disable(chan->phy);  in pxad_terminate_all()
1157 		pxad_free_phy(chan);  in pxad_terminate_all()
1158 		chan->phy = NULL;  in pxad_terminate_all()
1163 	spin_unlock_irqrestore(&chan->vc.lock, flags);  in pxad_terminate_all()
1164 	vchan_dma_desc_free_list(&chan->vc, &head);  in pxad_terminate_all()
1169 static unsigned int pxad_residue(struct pxad_chan *chan,  in pxad_residue()  argument
1184 	if (!chan->phy)  in pxad_residue()
1187 	spin_lock_irqsave(&chan->vc.lock, flags);  in pxad_residue()
1189 	vd = vchan_find_desc(&chan->vc, cookie);  in pxad_residue()
1195 		curr = phy_readl_relaxed(chan->phy, DSADR);  in pxad_residue()
1197 		curr = phy_readl_relaxed(chan->phy, DTADR);  in pxad_residue()
1238 	spin_unlock_irqrestore(&chan->vc.lock, flags);  in pxad_residue()
1239 	dev_dbg(&chan->vc.chan.dev->device,  in pxad_residue()
1249 	struct pxad_chan *chan = to_pxad_chan(dchan);  in pxad_tx_status()  local
1254 		dma_set_residue(txstate, pxad_residue(chan, cookie));  in pxad_tx_status()
1264 				 vc.chan.device_node) {  in pxad_free_channels()
1265 		list_del(&c->vc.chan.device_node);  in pxad_free_channels()
1334 	struct dma_chan *chan;  in pxad_dma_xlate()  local
1336 	chan = dma_get_any_slave_channel(&d->slave);  in pxad_dma_xlate()
1337 	if (!chan)  in pxad_dma_xlate()
1340 	to_pxad_chan(chan)->drcmr = dma_spec->args[0];  in pxad_dma_xlate()
1341 	to_pxad_chan(chan)->prio = dma_spec->args[1];  in pxad_dma_xlate()
1343 	return chan;  in pxad_dma_xlate()
1480 bool pxad_filter_fn(struct dma_chan *chan, void *param)  in pxad_filter_fn()  argument
1482 	struct pxad_chan *c = to_pxad_chan(chan);  in pxad_filter_fn()
1485 	if (chan->device->dev->driver != &pxad_driver.driver)  in pxad_filter_fn()