228 static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan)  in fsl_edma_enable_request()  argument
230 	void __iomem *addr = fsl_chan->edma->membase;  in fsl_edma_enable_request()
231 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_enable_request()
233 	edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), addr + EDMA_SEEI);  in fsl_edma_enable_request()
234 	edma_writeb(fsl_chan->edma, ch, addr + EDMA_SERQ);  in fsl_edma_enable_request()
237 static void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan)  in fsl_edma_disable_request()  argument
239 	void __iomem *addr = fsl_chan->edma->membase;  in fsl_edma_disable_request()
240 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_disable_request()
242 	edma_writeb(fsl_chan->edma, ch, addr + EDMA_CERQ);  in fsl_edma_disable_request()
243 	edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), addr + EDMA_CEEI);  in fsl_edma_disable_request()
246 static void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan,  in fsl_edma_chan_mux()  argument
249 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_chan_mux()
253 	chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR;  in fsl_edma_chan_mux()
254 	ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux;  in fsl_edma_chan_mux()
255 	muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux];  in fsl_edma_chan_mux()
294 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_terminate_all()  local
298 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_terminate_all()
299 	fsl_edma_disable_request(fsl_chan);  in fsl_edma_terminate_all()
300 	fsl_chan->edesc = NULL;  in fsl_edma_terminate_all()
301 	vchan_get_all_descriptors(&fsl_chan->vchan, &head);  in fsl_edma_terminate_all()
302 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_terminate_all()
303 	vchan_dma_desc_free_list(&fsl_chan->vchan, &head);  in fsl_edma_terminate_all()
309 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_pause()  local
312 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_pause()
313 	if (fsl_chan->edesc) {  in fsl_edma_pause()
314 		fsl_edma_disable_request(fsl_chan);  in fsl_edma_pause()
315 		fsl_chan->status = DMA_PAUSED;  in fsl_edma_pause()
317 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_pause()
323 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_resume()  local
326 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_resume()
327 	if (fsl_chan->edesc) {  in fsl_edma_resume()
328 		fsl_edma_enable_request(fsl_chan);  in fsl_edma_resume()
329 		fsl_chan->status = DMA_IN_PROGRESS;  in fsl_edma_resume()
331 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_resume()
338 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_slave_config()  local
340 	fsl_chan->fsc.dir = cfg->direction;  in fsl_edma_slave_config()
342 		fsl_chan->fsc.dev_addr = cfg->src_addr;  in fsl_edma_slave_config()
343 		fsl_chan->fsc.addr_width = cfg->src_addr_width;  in fsl_edma_slave_config()
344 		fsl_chan->fsc.burst = cfg->src_maxburst;  in fsl_edma_slave_config()
345 		fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->src_addr_width);  in fsl_edma_slave_config()
347 		fsl_chan->fsc.dev_addr = cfg->dst_addr;  in fsl_edma_slave_config()
348 		fsl_chan->fsc.addr_width = cfg->dst_addr_width;  in fsl_edma_slave_config()
349 		fsl_chan->fsc.burst = cfg->dst_maxburst;  in fsl_edma_slave_config()
350 		fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->dst_addr_width);  in fsl_edma_slave_config()
357 static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan,  in fsl_edma_desc_residue()  argument
360 	struct fsl_edma_desc *edesc = fsl_chan->edesc;  in fsl_edma_desc_residue()
361 	void __iomem *addr = fsl_chan->edma->membase;  in fsl_edma_desc_residue()
362 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_desc_residue()
363 	enum dma_transfer_direction dir = fsl_chan->fsc.dir;  in fsl_edma_desc_residue()
369 	for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++)  in fsl_edma_desc_residue()
377 		cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_SADDR(ch));  in fsl_edma_desc_residue()
379 		cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_DADDR(ch));  in fsl_edma_desc_residue()
382 	for (i = 0; i < fsl_chan->edesc->n_tcds; i++) {  in fsl_edma_desc_residue()
403 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_tx_status()  local
413 		return fsl_chan->status;  in fsl_edma_tx_status()
415 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_tx_status()
416 	vdesc = vchan_find_desc(&fsl_chan->vchan, cookie);  in fsl_edma_tx_status()
417 	if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie)  in fsl_edma_tx_status()
418 		txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, true);  in fsl_edma_tx_status()
420 		txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, false);  in fsl_edma_tx_status()
424 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_tx_status()
426 	return fsl_chan->status;  in fsl_edma_tx_status()
429 static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan,  in fsl_edma_set_tcd_regs()  argument
432 	struct fsl_edma_engine *edma = fsl_chan->edma;  in fsl_edma_set_tcd_regs()
433 	void __iomem *addr = fsl_chan->edma->membase;  in fsl_edma_set_tcd_regs()
434 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_set_tcd_regs()
502 static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan,  in fsl_edma_alloc_desc()  argument
513 	fsl_desc->echan = fsl_chan;  in fsl_edma_alloc_desc()
516 		fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool,  in fsl_edma_alloc_desc()
525 		dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd,  in fsl_edma_alloc_desc()
536 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_prep_dma_cyclic()  local
543 	if (!is_slave_direction(fsl_chan->fsc.dir))  in fsl_edma_prep_dma_cyclic()
547 	fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len);  in fsl_edma_prep_dma_cyclic()
553 	nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst;  in fsl_edma_prep_dma_cyclic()
563 		if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) {  in fsl_edma_prep_dma_cyclic()
565 			dst_addr = fsl_chan->fsc.dev_addr;  in fsl_edma_prep_dma_cyclic()
566 			soff = fsl_chan->fsc.addr_width;  in fsl_edma_prep_dma_cyclic()
569 			src_addr = fsl_chan->fsc.dev_addr;  in fsl_edma_prep_dma_cyclic()
572 			doff = fsl_chan->fsc.addr_width;  in fsl_edma_prep_dma_cyclic()
576 				  fsl_chan->fsc.attr, soff, nbytes, 0, iter,  in fsl_edma_prep_dma_cyclic()
581 	return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);  in fsl_edma_prep_dma_cyclic()
589 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_prep_slave_sg()  local
596 	if (!is_slave_direction(fsl_chan->fsc.dir))  in fsl_edma_prep_slave_sg()
599 	fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len);  in fsl_edma_prep_slave_sg()
604 	nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst;  in fsl_edma_prep_slave_sg()
609 		if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) {  in fsl_edma_prep_slave_sg()
611 			dst_addr = fsl_chan->fsc.dev_addr;  in fsl_edma_prep_slave_sg()
612 			soff = fsl_chan->fsc.addr_width;  in fsl_edma_prep_slave_sg()
615 			src_addr = fsl_chan->fsc.dev_addr;  in fsl_edma_prep_slave_sg()
618 			doff = fsl_chan->fsc.addr_width;  in fsl_edma_prep_slave_sg()
625 					  dst_addr, fsl_chan->fsc.attr, soff,  in fsl_edma_prep_slave_sg()
631 					  dst_addr, fsl_chan->fsc.attr, soff,  in fsl_edma_prep_slave_sg()
637 	return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);  in fsl_edma_prep_slave_sg()
640 static void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan)  in fsl_edma_xfer_desc()  argument
644 	vdesc = vchan_next_desc(&fsl_chan->vchan);  in fsl_edma_xfer_desc()
647 	fsl_chan->edesc = to_fsl_edma_desc(vdesc);  in fsl_edma_xfer_desc()
648 	fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd);  in fsl_edma_xfer_desc()
649 	fsl_edma_enable_request(fsl_chan);  in fsl_edma_xfer_desc()
650 	fsl_chan->status = DMA_IN_PROGRESS;  in fsl_edma_xfer_desc()
658 	struct fsl_edma_chan *fsl_chan;  in fsl_edma_tx_handler()  local
671 			fsl_chan = &fsl_edma->chans[ch];  in fsl_edma_tx_handler()
673 			spin_lock(&fsl_chan->vchan.lock);  in fsl_edma_tx_handler()
674 			if (!fsl_chan->edesc->iscyclic) {  in fsl_edma_tx_handler()
675 				list_del(&fsl_chan->edesc->vdesc.node);  in fsl_edma_tx_handler()
676 				vchan_cookie_complete(&fsl_chan->edesc->vdesc);  in fsl_edma_tx_handler()
677 				fsl_chan->edesc = NULL;  in fsl_edma_tx_handler()
678 				fsl_chan->status = DMA_COMPLETE;  in fsl_edma_tx_handler()
680 				vchan_cyclic_callback(&fsl_chan->edesc->vdesc);  in fsl_edma_tx_handler()
683 			if (!fsl_chan->edesc)  in fsl_edma_tx_handler()
684 				fsl_edma_xfer_desc(fsl_chan);  in fsl_edma_tx_handler()
686 			spin_unlock(&fsl_chan->vchan.lock);  in fsl_edma_tx_handler()
722 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_issue_pending()  local
725 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_issue_pending()
727 	if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc)  in fsl_edma_issue_pending()
728 		fsl_edma_xfer_desc(fsl_chan);  in fsl_edma_issue_pending()
730 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_issue_pending()
764 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_alloc_chan_resources()  local
766 	fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev,  in fsl_edma_alloc_chan_resources()
774 	struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan);  in fsl_edma_free_chan_resources()  local
778 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_free_chan_resources()
779 	fsl_edma_disable_request(fsl_chan);  in fsl_edma_free_chan_resources()
780 	fsl_edma_chan_mux(fsl_chan, 0, false);  in fsl_edma_free_chan_resources()
781 	fsl_chan->edesc = NULL;  in fsl_edma_free_chan_resources()
782 	vchan_get_all_descriptors(&fsl_chan->vchan, &head);  in fsl_edma_free_chan_resources()
783 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_free_chan_resources()
785 	vchan_dma_desc_free_list(&fsl_chan->vchan, &head);  in fsl_edma_free_chan_resources()
786 	dma_pool_destroy(fsl_chan->tcd_pool);  in fsl_edma_free_chan_resources()
787 	fsl_chan->tcd_pool = NULL;  in fsl_edma_free_chan_resources()
837 	struct fsl_edma_chan *fsl_chan;  in fsl_edma_probe()  local
848 	len = sizeof(*fsl_edma) + sizeof(*fsl_chan) * chans;  in fsl_edma_probe()
892 		struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i];  in fsl_edma_probe()  local
894 		fsl_chan->edma = fsl_edma;  in fsl_edma_probe()
896 		fsl_chan->vchan.desc_free = fsl_edma_free_desc;  in fsl_edma_probe()
897 		vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev);  in fsl_edma_probe()
900 		fsl_edma_chan_mux(fsl_chan, 0, false);  in fsl_edma_probe()