141 	struct dma_chan chan;  member
167 #define to_rcar_dmac_chan(c)	container_of(c, struct rcar_dmac_chan, chan)
290 static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg)  in rcar_dmac_chan_read()  argument
293 		return readw(chan->iomem + reg);  in rcar_dmac_chan_read()
295 		return readl(chan->iomem + reg);  in rcar_dmac_chan_read()
298 static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data)  in rcar_dmac_chan_write()  argument
301 		writew(data, chan->iomem + reg);  in rcar_dmac_chan_write()
303 		writel(data, chan->iomem + reg);  in rcar_dmac_chan_write()
310 static bool rcar_dmac_chan_is_busy(struct rcar_dmac_chan *chan)  in rcar_dmac_chan_is_busy()  argument
312 	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);  in rcar_dmac_chan_is_busy()
317 static void rcar_dmac_chan_start_xfer(struct rcar_dmac_chan *chan)  in rcar_dmac_chan_start_xfer()  argument
319 	struct rcar_dmac_desc *desc = chan->desc.running;  in rcar_dmac_chan_start_xfer()
322 	WARN_ON_ONCE(rcar_dmac_chan_is_busy(chan));  in rcar_dmac_chan_start_xfer()
324 	if (chan->mid_rid >= 0)  in rcar_dmac_chan_start_xfer()
325 		rcar_dmac_chan_write(chan, RCAR_DMARS, chan->mid_rid);  in rcar_dmac_chan_start_xfer()
330 		dev_dbg(chan->chan.device->dev,  in rcar_dmac_chan_start_xfer()
332 			chan->index, desc, desc->nchunks, &desc->hwdescs.dma);  in rcar_dmac_chan_start_xfer()
335 		rcar_dmac_chan_write(chan, RCAR_DMAFIXDPBASE,  in rcar_dmac_chan_start_xfer()
338 		rcar_dmac_chan_write(chan, RCAR_DMADPBASE,  in rcar_dmac_chan_start_xfer()
341 		rcar_dmac_chan_write(chan, RCAR_DMACHCRB,  in rcar_dmac_chan_start_xfer()
354 		rcar_dmac_chan_write(chan, RCAR_DMADAR,  in rcar_dmac_chan_start_xfer()
361 		rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(1));  in rcar_dmac_chan_start_xfer()
387 		dev_dbg(chan->chan.device->dev,  in rcar_dmac_chan_start_xfer()
389 			chan->index, chunk, chunk->size, &chunk->src_addr,  in rcar_dmac_chan_start_xfer()
393 		rcar_dmac_chan_write(chan, RCAR_DMAFIXSAR,  in rcar_dmac_chan_start_xfer()
395 		rcar_dmac_chan_write(chan, RCAR_DMAFIXDAR,  in rcar_dmac_chan_start_xfer()
398 		rcar_dmac_chan_write(chan, RCAR_DMASAR,  in rcar_dmac_chan_start_xfer()
400 		rcar_dmac_chan_write(chan, RCAR_DMADAR,  in rcar_dmac_chan_start_xfer()
402 		rcar_dmac_chan_write(chan, RCAR_DMATCR,  in rcar_dmac_chan_start_xfer()
408 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr | RCAR_DMACHCR_DE);  in rcar_dmac_chan_start_xfer()
435 	struct rcar_dmac_chan *chan = to_rcar_dmac_chan(tx->chan);  in rcar_dmac_tx_submit()  local
440 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_tx_submit()
444 	dev_dbg(chan->chan.device->dev, "chan%u: submit #%d@%p\n",  in rcar_dmac_tx_submit()
445 		chan->index, tx->cookie, desc);  in rcar_dmac_tx_submit()
447 	list_add_tail(&desc->node, &chan->desc.pending);  in rcar_dmac_tx_submit()
451 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_tx_submit()
465 static int rcar_dmac_desc_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)  in rcar_dmac_desc_alloc()  argument
479 		dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);  in rcar_dmac_desc_alloc()
486 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_desc_alloc()
487 	list_splice_tail(&list, &chan->desc.free);  in rcar_dmac_desc_alloc()
488 	list_add_tail(&page->node, &chan->desc.pages);  in rcar_dmac_desc_alloc()
489 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_desc_alloc()
506 static void rcar_dmac_desc_put(struct rcar_dmac_chan *chan,  in rcar_dmac_desc_put()  argument
511 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_desc_put()
512 	list_splice_tail_init(&desc->chunks, &chan->desc.chunks_free);  in rcar_dmac_desc_put()
513 	list_add_tail(&desc->node, &chan->desc.free);  in rcar_dmac_desc_put()
514 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_desc_put()
517 static void rcar_dmac_desc_recycle_acked(struct rcar_dmac_chan *chan)  in rcar_dmac_desc_recycle_acked()  argument
529 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_desc_recycle_acked()
530 	list_splice_init(&chan->desc.wait, &list);  in rcar_dmac_desc_recycle_acked()
531 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_desc_recycle_acked()
536 			rcar_dmac_desc_put(chan, desc);  in rcar_dmac_desc_recycle_acked()
544 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_desc_recycle_acked()
545 	list_splice(&list, &chan->desc.wait);  in rcar_dmac_desc_recycle_acked()
546 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_desc_recycle_acked()
558 static struct rcar_dmac_desc *rcar_dmac_desc_get(struct rcar_dmac_chan *chan)  in rcar_dmac_desc_get()  argument
565 	rcar_dmac_desc_recycle_acked(chan);  in rcar_dmac_desc_get()
567 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_desc_get()
569 	while (list_empty(&chan->desc.free)) {  in rcar_dmac_desc_get()
576 		spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_desc_get()
577 		ret = rcar_dmac_desc_alloc(chan, GFP_NOWAIT);  in rcar_dmac_desc_get()
580 		spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_desc_get()
583 	desc = list_first_entry(&chan->desc.free, struct rcar_dmac_desc, node);  in rcar_dmac_desc_get()
586 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_desc_get()
596 static int rcar_dmac_xfer_chunk_alloc(struct rcar_dmac_chan *chan, gfp_t gfp)  in rcar_dmac_xfer_chunk_alloc()  argument
613 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_xfer_chunk_alloc()
614 	list_splice_tail(&list, &chan->desc.chunks_free);  in rcar_dmac_xfer_chunk_alloc()
615 	list_add_tail(&page->node, &chan->desc.pages);  in rcar_dmac_xfer_chunk_alloc()
616 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_xfer_chunk_alloc()
631 rcar_dmac_xfer_chunk_get(struct rcar_dmac_chan *chan)  in rcar_dmac_xfer_chunk_get()  argument
637 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_xfer_chunk_get()
639 	while (list_empty(&chan->desc.chunks_free)) {  in rcar_dmac_xfer_chunk_get()
646 		spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_xfer_chunk_get()
647 		ret = rcar_dmac_xfer_chunk_alloc(chan, GFP_NOWAIT);  in rcar_dmac_xfer_chunk_get()
650 		spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_xfer_chunk_get()
653 	chunk = list_first_entry(&chan->desc.chunks_free,  in rcar_dmac_xfer_chunk_get()
657 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_xfer_chunk_get()
662 static void rcar_dmac_realloc_hwdesc(struct rcar_dmac_chan *chan,  in rcar_dmac_realloc_hwdesc()  argument
677 		dma_free_coherent(chan->chan.device->dev, desc->hwdescs.size,  in rcar_dmac_realloc_hwdesc()
686 	desc->hwdescs.mem = dma_alloc_coherent(chan->chan.device->dev, size,  in rcar_dmac_realloc_hwdesc()
694 static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan,  in rcar_dmac_fill_hwdesc()  argument
700 	rcar_dmac_realloc_hwdesc(chan, desc, desc->nchunks * sizeof(*hwdesc));  in rcar_dmac_fill_hwdesc()
720 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)  in rcar_dmac_chan_halt()  argument
722 	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);  in rcar_dmac_chan_halt()
726 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr);  in rcar_dmac_chan_halt()
729 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan)  in rcar_dmac_chan_reinit()  argument
735 	spin_lock_irqsave(&chan->lock, flags);  in rcar_dmac_chan_reinit()
738 	list_splice_init(&chan->desc.pending, &descs);  in rcar_dmac_chan_reinit()
739 	list_splice_init(&chan->desc.active, &descs);  in rcar_dmac_chan_reinit()
740 	list_splice_init(&chan->desc.done, &descs);  in rcar_dmac_chan_reinit()
741 	list_splice_init(&chan->desc.wait, &descs);  in rcar_dmac_chan_reinit()
743 	chan->desc.running = NULL;  in rcar_dmac_chan_reinit()
745 	spin_unlock_irqrestore(&chan->lock, flags);  in rcar_dmac_chan_reinit()
749 		rcar_dmac_desc_put(chan, desc);  in rcar_dmac_chan_reinit()
764 		struct rcar_dmac_chan *chan = &dmac->channels[i];  in rcar_dmac_abort()  local
767 		spin_lock(&chan->lock);  in rcar_dmac_abort()
768 		rcar_dmac_chan_halt(chan);  in rcar_dmac_abort()
769 		spin_unlock(&chan->lock);  in rcar_dmac_abort()
771 		rcar_dmac_chan_reinit(chan);  in rcar_dmac_abort()
779 static void rcar_dmac_chan_configure_desc(struct rcar_dmac_chan *chan,  in rcar_dmac_chan_configure_desc()  argument
796 		xfer_size = chan->src_xfer_size;  in rcar_dmac_chan_configure_desc()
802 		xfer_size = chan->dst_xfer_size;  in rcar_dmac_chan_configure_desc()
828 rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl,  in rcar_dmac_chan_prep_sg()  argument
842 	desc = rcar_dmac_desc_get(chan);  in rcar_dmac_chan_prep_sg()
852 	rcar_dmac_chan_configure_desc(chan, desc);  in rcar_dmac_chan_prep_sg()
888 			chunk = rcar_dmac_xfer_chunk_get(chan);  in rcar_dmac_chan_prep_sg()
890 				rcar_dmac_desc_put(chan, desc);  in rcar_dmac_chan_prep_sg()
904 			dev_dbg(chan->chan.device->dev,  in rcar_dmac_chan_prep_sg()
906 				chan->index, chunk, desc, i, sg, size, len,  in rcar_dmac_chan_prep_sg()
935 		if (rcar_dmac_fill_hwdesc(chan, desc) < 0)  in rcar_dmac_chan_prep_sg()
946 static int rcar_dmac_alloc_chan_resources(struct dma_chan *chan)  in rcar_dmac_alloc_chan_resources()  argument
948 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_alloc_chan_resources()
963 	return pm_runtime_get_sync(chan->device->dev);  in rcar_dmac_alloc_chan_resources()
966 static void rcar_dmac_free_chan_resources(struct dma_chan *chan)  in rcar_dmac_free_chan_resources()  argument
968 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_free_chan_resources()
969 	struct rcar_dmac *dmac = to_rcar_dmac(chan->device);  in rcar_dmac_free_chan_resources()
1001 	pm_runtime_put(chan->device->dev);  in rcar_dmac_free_chan_resources()
1005 rcar_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,  in rcar_dmac_prep_dma_memcpy()  argument
1008 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_prep_dma_memcpy()
1025 rcar_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,  in rcar_dmac_prep_slave_sg()  argument
1029 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_prep_slave_sg()
1034 		dev_warn(chan->device->dev,  in rcar_dmac_prep_slave_sg()
1049 rcar_dmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,  in rcar_dmac_prep_dma_cyclic()  argument
1053 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_prep_dma_cyclic()
1062 		dev_warn(chan->device->dev,  in rcar_dmac_prep_dma_cyclic()
1070 		dev_err(chan->device->dev,  in rcar_dmac_prep_dma_cyclic()
1104 static int rcar_dmac_device_config(struct dma_chan *chan,  in rcar_dmac_device_config()  argument
1107 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_device_config()
1121 static int rcar_dmac_chan_terminate_all(struct dma_chan *chan)  in rcar_dmac_chan_terminate_all()  argument
1123 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_chan_terminate_all()
1140 static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan,  in rcar_dmac_chan_get_residue()  argument
1143 	struct rcar_dmac_desc *desc = chan->desc.running;  in rcar_dmac_chan_get_residue()
1167 		dptr = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &  in rcar_dmac_chan_get_residue()
1183 	residue += rcar_dmac_chan_read(chan, RCAR_DMATCR) << desc->xfer_shift;  in rcar_dmac_chan_get_residue()
1188 static enum dma_status rcar_dmac_tx_status(struct dma_chan *chan,  in rcar_dmac_tx_status()  argument
1192 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_tx_status()
1197 	status = dma_cookie_status(chan, cookie, txstate);  in rcar_dmac_tx_status()
1210 static void rcar_dmac_issue_pending(struct dma_chan *chan)  in rcar_dmac_issue_pending()  argument
1212 	struct rcar_dmac_chan *rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_issue_pending()
1245 static irqreturn_t rcar_dmac_isr_desc_stage_end(struct rcar_dmac_chan *chan)  in rcar_dmac_isr_desc_stage_end()  argument
1247 	struct rcar_dmac_desc *desc = chan->desc.running;  in rcar_dmac_isr_desc_stage_end()
1260 	stage = (rcar_dmac_chan_read(chan, RCAR_DMACHCRB) &  in rcar_dmac_isr_desc_stage_end()
1262 	rcar_dmac_chan_write(chan, RCAR_DMADPCR, RCAR_DMADPCR_DIPT(stage));  in rcar_dmac_isr_desc_stage_end()
1267 static irqreturn_t rcar_dmac_isr_transfer_end(struct rcar_dmac_chan *chan)  in rcar_dmac_isr_transfer_end()  argument
1269 	struct rcar_dmac_desc *desc = chan->desc.running;  in rcar_dmac_isr_transfer_end()
1313 	list_move_tail(&desc->node, &chan->desc.done);  in rcar_dmac_isr_transfer_end()
1316 	if (!list_empty(&chan->desc.active))  in rcar_dmac_isr_transfer_end()
1317 		chan->desc.running = list_first_entry(&chan->desc.active,  in rcar_dmac_isr_transfer_end()
1321 		chan->desc.running = NULL;  in rcar_dmac_isr_transfer_end()
1324 	if (chan->desc.running)  in rcar_dmac_isr_transfer_end()
1325 		rcar_dmac_chan_start_xfer(chan);  in rcar_dmac_isr_transfer_end()
1333 	struct rcar_dmac_chan *chan = dev;  in rcar_dmac_isr_channel()  local
1337 	spin_lock(&chan->lock);  in rcar_dmac_isr_channel()
1339 	chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);  in rcar_dmac_isr_channel()
1342 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask);  in rcar_dmac_isr_channel()
1345 		ret |= rcar_dmac_isr_desc_stage_end(chan);  in rcar_dmac_isr_channel()
1348 		ret |= rcar_dmac_isr_transfer_end(chan);  in rcar_dmac_isr_channel()
1350 	spin_unlock(&chan->lock);  in rcar_dmac_isr_channel()
1357 	struct rcar_dmac_chan *chan = dev;  in rcar_dmac_isr_channel_thread()  local
1360 	spin_lock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1363 	if (chan->desc.running && chan->desc.running->cyclic) {  in rcar_dmac_isr_channel_thread()
1367 		desc = chan->desc.running;  in rcar_dmac_isr_channel_thread()
1372 			spin_unlock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1374 			spin_lock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1382 	while (!list_empty(&chan->desc.done)) {  in rcar_dmac_isr_channel_thread()
1383 		desc = list_first_entry(&chan->desc.done, struct rcar_dmac_desc,  in rcar_dmac_isr_channel_thread()
1389 			spin_unlock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1396 			spin_lock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1399 		list_add_tail(&desc->node, &chan->desc.wait);  in rcar_dmac_isr_channel_thread()
1402 	spin_unlock_irq(&chan->lock);  in rcar_dmac_isr_channel_thread()
1405 	rcar_dmac_desc_recycle_acked(chan);  in rcar_dmac_isr_channel_thread()
1432 static bool rcar_dmac_chan_filter(struct dma_chan *chan, void *arg)  in rcar_dmac_chan_filter()  argument
1434 	struct rcar_dmac *dmac = to_rcar_dmac(chan->device);  in rcar_dmac_chan_filter()
1444 	if (chan->device->device_config != rcar_dmac_device_config ||  in rcar_dmac_chan_filter()
1445 	    dma_spec->np != chan->device->dev->of_node)  in rcar_dmac_chan_filter()
1455 	struct dma_chan *chan;  in rcar_dmac_of_xlate()  local
1465 	chan = dma_request_channel(mask, rcar_dmac_chan_filter, dma_spec);  in rcar_dmac_of_xlate()
1466 	if (!chan)  in rcar_dmac_of_xlate()
1469 	rchan = to_rcar_dmac_chan(chan);  in rcar_dmac_of_xlate()
1472 	return chan;  in rcar_dmac_of_xlate()
1524 	struct dma_chan *chan = &rchan->chan;  in rcar_dmac_chan_probe()  local
1567 	chan->device = &dmac->engine;  in rcar_dmac_chan_probe()
1568 	dma_cookie_init(chan);  in rcar_dmac_chan_probe()
1570 	list_add_tail(&chan->device_node, &dmac->engine.channels);  in rcar_dmac_chan_probe()