131 	void (*enable_chan)(struct mdc_chan *mchan);
132 	void (*disable_chan)(struct mdc_chan *mchan);
161 static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)  in mdc_chan_readl()  argument
163 	return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);  in mdc_chan_readl()
166 static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)  in mdc_chan_writel()  argument
168 	mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);  in mdc_chan_writel()
207 static void mdc_list_desc_config(struct mdc_chan *mchan,  in mdc_list_desc_config()  argument
212 	struct mdc_dma *mdma = mchan->mdma;  in mdc_list_desc_config()
219 		(mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |  in mdc_list_desc_config()
220 		(mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |  in mdc_list_desc_config()
221 		(mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);  in mdc_list_desc_config()
241 		mdc_set_write_width(ldesc, mchan->config.dst_addr_width);  in mdc_list_desc_config()
242 		burst_size = min(max_burst, mchan->config.dst_maxburst *  in mdc_list_desc_config()
243 				 mchan->config.dst_addr_width);  in mdc_list_desc_config()
247 		mdc_set_read_width(ldesc, mchan->config.src_addr_width);  in mdc_list_desc_config()
249 		burst_size = min(max_burst, mchan->config.src_maxburst *  in mdc_list_desc_config()
250 				 mchan->config.src_addr_width);  in mdc_list_desc_config()
291 	struct mdc_chan *mchan = to_mdc_chan(chan);  in mdc_prep_dma_memcpy()  local
292 	struct mdc_dma *mdma = mchan->mdma;  in mdc_prep_dma_memcpy()
303 	mdesc->chan = mchan;  in mdc_prep_dma_memcpy()
323 		mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,  in mdc_prep_dma_memcpy()
335 	return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);  in mdc_prep_dma_memcpy()
343 static int mdc_check_slave_width(struct mdc_chan *mchan,  in mdc_check_slave_width()  argument
349 		width = mchan->config.dst_addr_width;  in mdc_check_slave_width()
351 		width = mchan->config.src_addr_width;  in mdc_check_slave_width()
363 	if (width > mchan->mdma->bus_width)  in mdc_check_slave_width()
374 	struct mdc_chan *mchan = to_mdc_chan(chan);  in mdc_prep_dma_cyclic()  local
375 	struct mdc_dma *mdma = mchan->mdma;  in mdc_prep_dma_cyclic()
386 	if (mdc_check_slave_width(mchan, dir) < 0)  in mdc_prep_dma_cyclic()
392 	mdesc->chan = mchan;  in mdc_prep_dma_cyclic()
421 				mdc_list_desc_config(mchan, curr, dir,  in mdc_prep_dma_cyclic()
423 						     mchan->config.dst_addr,  in mdc_prep_dma_cyclic()
426 				mdc_list_desc_config(mchan, curr, dir,  in mdc_prep_dma_cyclic()
427 						     mchan->config.src_addr,  in mdc_prep_dma_cyclic()
443 	return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);  in mdc_prep_dma_cyclic()
456 	struct mdc_chan *mchan = to_mdc_chan(chan);  in mdc_prep_slave_sg()  local
457 	struct mdc_dma *mdma = mchan->mdma;  in mdc_prep_slave_sg()
470 	if (mdc_check_slave_width(mchan, dir) < 0)  in mdc_prep_slave_sg()
476 	mdesc->chan = mchan;  in mdc_prep_slave_sg()
502 				mdc_list_desc_config(mchan, curr, dir, buf,  in mdc_prep_slave_sg()
503 						     mchan->config.dst_addr,  in mdc_prep_slave_sg()
506 				mdc_list_desc_config(mchan, curr, dir,  in mdc_prep_slave_sg()
507 						     mchan->config.src_addr,  in mdc_prep_slave_sg()
521 	return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);  in mdc_prep_slave_sg()
529 static void mdc_issue_desc(struct mdc_chan *mchan)  in mdc_issue_desc()  argument
531 	struct mdc_dma *mdma = mchan->mdma;  in mdc_issue_desc()
536 	vd = vchan_next_desc(&mchan->vc);  in mdc_issue_desc()
543 	mchan->desc = mdesc;  in mdc_issue_desc()
546 		mchan->chan_nr);  in mdc_issue_desc()
548 	mdma->soc->enable_chan(mchan);  in mdc_issue_desc()
550 	val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);  in mdc_issue_desc()
554 	mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);  in mdc_issue_desc()
555 	val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |  in mdc_issue_desc()
556 		(mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |  in mdc_issue_desc()
557 		(mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);  in mdc_issue_desc()
558 	mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);  in mdc_issue_desc()
559 	mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);  in mdc_issue_desc()
560 	val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);  in mdc_issue_desc()
562 	mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);  in mdc_issue_desc()
567 	struct mdc_chan *mchan = to_mdc_chan(chan);  in mdc_issue_pending()  local
570 	spin_lock_irqsave(&mchan->vc.lock, flags);  in mdc_issue_pending()
571 	if (vchan_issue_pending(&mchan->vc) && !mchan->desc)  in mdc_issue_pending()
572 		mdc_issue_desc(mchan);  in mdc_issue_pending()
573 	spin_unlock_irqrestore(&mchan->vc.lock, flags);  in mdc_issue_pending()
579 	struct mdc_chan *mchan = to_mdc_chan(chan);  in mdc_tx_status()  local
593 	spin_lock_irqsave(&mchan->vc.lock, flags);  in mdc_tx_status()
594 	vd = vchan_find_desc(&mchan->vc, cookie);  in mdc_tx_status()
598 	} else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {  in mdc_tx_status()
603 		mdesc = mchan->desc;  in mdc_tx_status()
610 			val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &  in mdc_tx_status()
612 			residue = mdc_chan_readl(mchan,  in mdc_tx_status()
614 			val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &  in mdc_tx_status()
647 	spin_unlock_irqrestore(&mchan->vc.lock, flags);  in mdc_tx_status()
656 	struct mdc_chan *mchan = to_mdc_chan(chan);  in mdc_terminate_all()  local
661 	spin_lock_irqsave(&mchan->vc.lock, flags);  in mdc_terminate_all()
663 	mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,  in mdc_terminate_all()
666 	mdesc = mchan->desc;  in mdc_terminate_all()
667 	mchan->desc = NULL;  in mdc_terminate_all()
668 	vchan_get_all_descriptors(&mchan->vc, &head);  in mdc_terminate_all()
670 	spin_unlock_irqrestore(&mchan->vc.lock, flags);  in mdc_terminate_all()
674 	vchan_dma_desc_free_list(&mchan->vc, &head);  in mdc_terminate_all()
682 	struct mdc_chan *mchan = to_mdc_chan(chan);  in mdc_slave_config()  local
685 	spin_lock_irqsave(&mchan->vc.lock, flags);  in mdc_slave_config()
686 	mchan->config = *config;  in mdc_slave_config()
687 	spin_unlock_irqrestore(&mchan->vc.lock, flags);  in mdc_slave_config()
694 	struct mdc_chan *mchan = to_mdc_chan(chan);  in mdc_free_chan_resources()  local
695 	struct mdc_dma *mdma = mchan->mdma;  in mdc_free_chan_resources()
699 	mdma->soc->disable_chan(mchan);  in mdc_free_chan_resources()
704 	struct mdc_chan *mchan = (struct mdc_chan *)dev_id;  in mdc_chan_irq()  local
709 	spin_lock(&mchan->vc.lock);  in mdc_chan_irq()
711 	val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);  in mdc_chan_irq()
720 		val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);  in mdc_chan_irq()
727 		mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);  in mdc_chan_irq()
728 		val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);  in mdc_chan_irq()
733 	dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);  in mdc_chan_irq()
735 	mdesc = mchan->desc;  in mdc_chan_irq()
737 		dev_warn(mdma2dev(mchan->mdma),  in mdc_chan_irq()
739 			 mchan->chan_nr);  in mdc_chan_irq()
761 			mchan->desc = NULL;  in mdc_chan_irq()
763 			mdc_issue_desc(mchan);  in mdc_chan_irq()
768 	spin_unlock(&mchan->vc.lock);  in mdc_chan_irq()
783 		struct mdc_chan *mchan = to_mdc_chan(chan);  in mdc_of_xlate()  local
785 		if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))  in mdc_of_xlate()
788 			mchan->periph = dma_spec->args[0];  in mdc_of_xlate()
789 			mchan->thread = dma_spec->args[2];  in mdc_of_xlate()
801 static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)  in pistachio_mdc_enable_chan()  argument
803 	struct mdc_dma *mdma = mchan->mdma;  in pistachio_mdc_enable_chan()
806 			   PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),  in pistachio_mdc_enable_chan()
808 			   PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),  in pistachio_mdc_enable_chan()
809 			   mchan->periph <<  in pistachio_mdc_enable_chan()
810 			   PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));  in pistachio_mdc_enable_chan()
813 static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)  in pistachio_mdc_disable_chan()  argument
815 	struct mdc_dma *mdma = mchan->mdma;  in pistachio_mdc_disable_chan()
818 			   PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),  in pistachio_mdc_disable_chan()
820 			   PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),  in pistachio_mdc_disable_chan()
923 		struct mdc_chan *mchan = &mdma->channels[i];  in mdc_dma_probe()  local
925 		mchan->mdma = mdma;  in mdc_dma_probe()
926 		mchan->chan_nr = i;  in mdc_dma_probe()
927 		mchan->irq = platform_get_irq(pdev, i);  in mdc_dma_probe()
928 		if (mchan->irq < 0) {  in mdc_dma_probe()
929 			ret = mchan->irq;  in mdc_dma_probe()
932 		ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,  in mdc_dma_probe()
934 				       dev_name(&pdev->dev), mchan);  in mdc_dma_probe()
938 		mchan->vc.desc_free = mdc_desc_free;  in mdc_dma_probe()
939 		vchan_init(&mchan->vc, &mdma->dma_dev);  in mdc_dma_probe()
973 	struct mdc_chan *mchan, *next;  in mdc_dma_remove()  local
978 	list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,  in mdc_dma_remove()
980 		list_del(&mchan->vc.chan.device_node);  in mdc_dma_remove()
982 		synchronize_irq(mchan->irq);  in mdc_dma_remove()
983 		devm_free_irq(&pdev->dev, mchan->irq, mchan);  in mdc_dma_remove()
985 		tasklet_kill(&mchan->vc.task);  in mdc_dma_remove()