247 	struct xilinx_vdma_chan *chan[XILINX_VDMA_MAX_CHANS_PER_DEVICE];  member
253 #define to_xilinx_chan(chan) \  argument
254 	container_of(chan, struct xilinx_vdma_chan, common)
259 static inline u32 vdma_read(struct xilinx_vdma_chan *chan, u32 reg)  in vdma_read()  argument
261 	return ioread32(chan->xdev->regs + reg);  in vdma_read()
264 static inline void vdma_write(struct xilinx_vdma_chan *chan, u32 reg, u32 value)  in vdma_write()  argument
266 	iowrite32(value, chan->xdev->regs + reg);  in vdma_write()
269 static inline void vdma_desc_write(struct xilinx_vdma_chan *chan, u32 reg,  in vdma_desc_write()  argument
272 	vdma_write(chan, chan->desc_offset + reg, value);  in vdma_desc_write()
275 static inline u32 vdma_ctrl_read(struct xilinx_vdma_chan *chan, u32 reg)  in vdma_ctrl_read()  argument
277 	return vdma_read(chan, chan->ctrl_offset + reg);  in vdma_ctrl_read()
280 static inline void vdma_ctrl_write(struct xilinx_vdma_chan *chan, u32 reg,  in vdma_ctrl_write()  argument
283 	vdma_write(chan, chan->ctrl_offset + reg, value);  in vdma_ctrl_write()
286 static inline void vdma_ctrl_clr(struct xilinx_vdma_chan *chan, u32 reg,  in vdma_ctrl_clr()  argument
289 	vdma_ctrl_write(chan, reg, vdma_ctrl_read(chan, reg) & ~clr);  in vdma_ctrl_clr()
292 static inline void vdma_ctrl_set(struct xilinx_vdma_chan *chan, u32 reg,  in vdma_ctrl_set()  argument
295 	vdma_ctrl_write(chan, reg, vdma_ctrl_read(chan, reg) | set);  in vdma_ctrl_set()
309 xilinx_vdma_alloc_tx_segment(struct xilinx_vdma_chan *chan)  in xilinx_vdma_alloc_tx_segment()  argument
314 	segment = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &phys);  in xilinx_vdma_alloc_tx_segment()
329 static void xilinx_vdma_free_tx_segment(struct xilinx_vdma_chan *chan,  in xilinx_vdma_free_tx_segment()  argument
332 	dma_pool_free(chan->desc_pool, segment, segment->phys);  in xilinx_vdma_free_tx_segment()
342 xilinx_vdma_alloc_tx_descriptor(struct xilinx_vdma_chan *chan)  in xilinx_vdma_alloc_tx_descriptor()  argument
347 	if (chan->allocated_desc)  in xilinx_vdma_alloc_tx_descriptor()
348 		return chan->allocated_desc;  in xilinx_vdma_alloc_tx_descriptor()
354 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_vdma_alloc_tx_descriptor()
355 	chan->allocated_desc = desc;  in xilinx_vdma_alloc_tx_descriptor()
356 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_vdma_alloc_tx_descriptor()
369 xilinx_vdma_free_tx_descriptor(struct xilinx_vdma_chan *chan,  in xilinx_vdma_free_tx_descriptor()  argument
379 		xilinx_vdma_free_tx_segment(chan, segment);  in xilinx_vdma_free_tx_descriptor()
392 static void xilinx_vdma_free_desc_list(struct xilinx_vdma_chan *chan,  in xilinx_vdma_free_desc_list()  argument
399 		xilinx_vdma_free_tx_descriptor(chan, desc);  in xilinx_vdma_free_desc_list()
407 static void xilinx_vdma_free_descriptors(struct xilinx_vdma_chan *chan)  in xilinx_vdma_free_descriptors()  argument
411 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_vdma_free_descriptors()
413 	xilinx_vdma_free_desc_list(chan, &chan->pending_list);  in xilinx_vdma_free_descriptors()
414 	xilinx_vdma_free_desc_list(chan, &chan->done_list);  in xilinx_vdma_free_descriptors()
416 	xilinx_vdma_free_tx_descriptor(chan, chan->active_desc);  in xilinx_vdma_free_descriptors()
417 	chan->active_desc = NULL;  in xilinx_vdma_free_descriptors()
419 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_vdma_free_descriptors()
428 	struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan);  in xilinx_vdma_free_chan_resources()  local
430 	dev_dbg(chan->dev, "Free all channel resources.\n");  in xilinx_vdma_free_chan_resources()
432 	xilinx_vdma_free_descriptors(chan);  in xilinx_vdma_free_chan_resources()
433 	dma_pool_destroy(chan->desc_pool);  in xilinx_vdma_free_chan_resources()
434 	chan->desc_pool = NULL;  in xilinx_vdma_free_chan_resources()
441 static void xilinx_vdma_chan_desc_cleanup(struct xilinx_vdma_chan *chan)  in xilinx_vdma_chan_desc_cleanup()  argument
446 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_vdma_chan_desc_cleanup()
448 	list_for_each_entry_safe(desc, next, &chan->done_list, node) {  in xilinx_vdma_chan_desc_cleanup()
459 			spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_vdma_chan_desc_cleanup()
461 			spin_lock_irqsave(&chan->lock, flags);  in xilinx_vdma_chan_desc_cleanup()
466 		xilinx_vdma_free_tx_descriptor(chan, desc);  in xilinx_vdma_chan_desc_cleanup()
469 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_vdma_chan_desc_cleanup()
478 	struct xilinx_vdma_chan *chan = (struct xilinx_vdma_chan *)data;  in xilinx_vdma_do_tasklet()  local
480 	xilinx_vdma_chan_desc_cleanup(chan);  in xilinx_vdma_do_tasklet()
491 	struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan);  in xilinx_vdma_alloc_chan_resources()  local
494 	if (chan->desc_pool)  in xilinx_vdma_alloc_chan_resources()
501 	chan->desc_pool = dma_pool_create("xilinx_vdma_desc_pool",  in xilinx_vdma_alloc_chan_resources()
502 				chan->dev,  in xilinx_vdma_alloc_chan_resources()
505 	if (!chan->desc_pool) {  in xilinx_vdma_alloc_chan_resources()
506 		dev_err(chan->dev,  in xilinx_vdma_alloc_chan_resources()
508 			chan->id);  in xilinx_vdma_alloc_chan_resources()
537 static bool xilinx_vdma_is_running(struct xilinx_vdma_chan *chan)  in xilinx_vdma_is_running()  argument
539 	return !(vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) &  in xilinx_vdma_is_running()
541 		(vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR) &  in xilinx_vdma_is_running()
551 static bool xilinx_vdma_is_idle(struct xilinx_vdma_chan *chan)  in xilinx_vdma_is_idle()  argument
553 	return vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) &  in xilinx_vdma_is_idle()
561 static void xilinx_vdma_halt(struct xilinx_vdma_chan *chan)  in xilinx_vdma_halt()  argument
565 	vdma_ctrl_clr(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RUNSTOP);  in xilinx_vdma_halt()
569 		if (vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) &  in xilinx_vdma_halt()
575 		dev_err(chan->dev, "Cannot stop channel %p: %x\n",  in xilinx_vdma_halt()
576 			chan, vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR));  in xilinx_vdma_halt()
577 		chan->err = true;  in xilinx_vdma_halt()
587 static void xilinx_vdma_start(struct xilinx_vdma_chan *chan)  in xilinx_vdma_start()  argument
591 	vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RUNSTOP);  in xilinx_vdma_start()
595 		if (!(vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) &  in xilinx_vdma_start()
601 		dev_err(chan->dev, "Cannot start channel %p: %x\n",  in xilinx_vdma_start()
602 			chan, vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR));  in xilinx_vdma_start()
604 		chan->err = true;  in xilinx_vdma_start()
614 static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan)  in xilinx_vdma_start_transfer()  argument
616 	struct xilinx_vdma_config *config = &chan->config;  in xilinx_vdma_start_transfer()
622 	if (chan->err)  in xilinx_vdma_start_transfer()
625 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_vdma_start_transfer()
628 	if (chan->active_desc)  in xilinx_vdma_start_transfer()
631 	if (list_empty(&chan->pending_list))  in xilinx_vdma_start_transfer()
634 	desc = list_first_entry(&chan->pending_list,  in xilinx_vdma_start_transfer()
638 	if (chan->has_sg && xilinx_vdma_is_running(chan) &&  in xilinx_vdma_start_transfer()
639 	    !xilinx_vdma_is_idle(chan)) {  in xilinx_vdma_start_transfer()
640 		dev_dbg(chan->dev, "DMA controller still busy\n");  in xilinx_vdma_start_transfer()
648 	if (chan->has_sg) {  in xilinx_vdma_start_transfer()
654 		vdma_ctrl_write(chan, XILINX_VDMA_REG_CURDESC, head->phys);  in xilinx_vdma_start_transfer()
658 	reg = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR);  in xilinx_vdma_start_transfer()
669 	if (chan->has_sg || !config->park)  in xilinx_vdma_start_transfer()
675 	vdma_ctrl_write(chan, XILINX_VDMA_REG_DMACR, reg);  in xilinx_vdma_start_transfer()
678 			(config->park_frm < chan->num_frms)) {  in xilinx_vdma_start_transfer()
679 		if (chan->direction == DMA_MEM_TO_DEV)  in xilinx_vdma_start_transfer()
680 			vdma_write(chan, XILINX_VDMA_REG_PARK_PTR,  in xilinx_vdma_start_transfer()
684 			vdma_write(chan, XILINX_VDMA_REG_PARK_PTR,  in xilinx_vdma_start_transfer()
690 	xilinx_vdma_start(chan);  in xilinx_vdma_start_transfer()
692 	if (chan->err)  in xilinx_vdma_start_transfer()
696 	if (chan->has_sg) {  in xilinx_vdma_start_transfer()
697 		vdma_ctrl_write(chan, XILINX_VDMA_REG_TAILDESC, tail->phys);  in xilinx_vdma_start_transfer()
703 			vdma_desc_write(chan,  in xilinx_vdma_start_transfer()
713 		vdma_desc_write(chan, XILINX_VDMA_REG_HSIZE, last->hw.hsize);  in xilinx_vdma_start_transfer()
714 		vdma_desc_write(chan, XILINX_VDMA_REG_FRMDLY_STRIDE,  in xilinx_vdma_start_transfer()
716 		vdma_desc_write(chan, XILINX_VDMA_REG_VSIZE, last->hw.vsize);  in xilinx_vdma_start_transfer()
720 	chan->active_desc = desc;  in xilinx_vdma_start_transfer()
723 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_vdma_start_transfer()
732 	struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan);  in xilinx_vdma_issue_pending()  local
734 	xilinx_vdma_start_transfer(chan);  in xilinx_vdma_issue_pending()
743 static void xilinx_vdma_complete_descriptor(struct xilinx_vdma_chan *chan)  in xilinx_vdma_complete_descriptor()  argument
748 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_vdma_complete_descriptor()
750 	desc = chan->active_desc;  in xilinx_vdma_complete_descriptor()
752 		dev_dbg(chan->dev, "no running descriptors\n");  in xilinx_vdma_complete_descriptor()
757 	list_add_tail(&desc->node, &chan->done_list);  in xilinx_vdma_complete_descriptor()
759 	chan->active_desc = NULL;  in xilinx_vdma_complete_descriptor()
762 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_vdma_complete_descriptor()
771 static int xilinx_vdma_reset(struct xilinx_vdma_chan *chan)  in xilinx_vdma_reset()  argument
776 	vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RESET);  in xilinx_vdma_reset()
778 	tmp = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR) &  in xilinx_vdma_reset()
783 		tmp = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR) &  in xilinx_vdma_reset()
788 		dev_err(chan->dev, "reset timeout, cr %x, sr %x\n",  in xilinx_vdma_reset()
789 			vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR),  in xilinx_vdma_reset()
790 			vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR));  in xilinx_vdma_reset()
794 	chan->err = false;  in xilinx_vdma_reset()
805 static int xilinx_vdma_chan_reset(struct xilinx_vdma_chan *chan)  in xilinx_vdma_chan_reset()  argument
810 	err = xilinx_vdma_reset(chan);  in xilinx_vdma_chan_reset()
815 	vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR,  in xilinx_vdma_chan_reset()
830 	struct xilinx_vdma_chan *chan = data;  in xilinx_vdma_irq_handler()  local
834 	status = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR);  in xilinx_vdma_irq_handler()
838 	vdma_ctrl_write(chan, XILINX_VDMA_REG_DMASR,  in xilinx_vdma_irq_handler()
850 		vdma_ctrl_write(chan, XILINX_VDMA_REG_DMASR,  in xilinx_vdma_irq_handler()
853 		if (!chan->flush_on_fsync ||  in xilinx_vdma_irq_handler()
855 			dev_err(chan->dev,  in xilinx_vdma_irq_handler()
857 				chan, errors,  in xilinx_vdma_irq_handler()
858 				vdma_ctrl_read(chan, XILINX_VDMA_REG_CURDESC),  in xilinx_vdma_irq_handler()
859 				vdma_ctrl_read(chan, XILINX_VDMA_REG_TAILDESC));  in xilinx_vdma_irq_handler()
860 			chan->err = true;  in xilinx_vdma_irq_handler()
869 		dev_dbg(chan->dev, "Inter-packet latency too long\n");  in xilinx_vdma_irq_handler()
873 		xilinx_vdma_complete_descriptor(chan);  in xilinx_vdma_irq_handler()
874 		xilinx_vdma_start_transfer(chan);  in xilinx_vdma_irq_handler()
877 	tasklet_schedule(&chan->tasklet);  in xilinx_vdma_irq_handler()
890 	struct xilinx_vdma_chan *chan = to_xilinx_chan(tx->chan);  in xilinx_vdma_tx_submit()  local
895 	if (chan->err) {  in xilinx_vdma_tx_submit()
900 		err = xilinx_vdma_chan_reset(chan);  in xilinx_vdma_tx_submit()
905 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_vdma_tx_submit()
910 	list_add_tail(&desc->node, &chan->pending_list);  in xilinx_vdma_tx_submit()
913 	chan->allocated_desc = NULL;  in xilinx_vdma_tx_submit()
915 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_vdma_tx_submit()
934 	struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan);  in xilinx_vdma_dma_prep_interleaved()  local
949 	desc = xilinx_vdma_alloc_tx_descriptor(chan);  in xilinx_vdma_dma_prep_interleaved()
953 	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);  in xilinx_vdma_dma_prep_interleaved()
958 	segment = xilinx_vdma_alloc_tx_segment(chan);  in xilinx_vdma_dma_prep_interleaved()
968 	hw->stride |= chan->config.frm_dly <<  in xilinx_vdma_dma_prep_interleaved()
996 	xilinx_vdma_free_tx_descriptor(chan, desc);  in xilinx_vdma_dma_prep_interleaved()
1006 	struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan);  in xilinx_vdma_terminate_all()  local
1009 	xilinx_vdma_halt(chan);  in xilinx_vdma_terminate_all()
1012 	xilinx_vdma_free_descriptors(chan);  in xilinx_vdma_terminate_all()
1033 	struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan);  in xilinx_vdma_channel_set_config()  local
1037 		return xilinx_vdma_chan_reset(chan);  in xilinx_vdma_channel_set_config()
1039 	dmacr = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR);  in xilinx_vdma_channel_set_config()
1041 	chan->config.frm_dly = cfg->frm_dly;  in xilinx_vdma_channel_set_config()
1042 	chan->config.park = cfg->park;  in xilinx_vdma_channel_set_config()
1045 	chan->config.gen_lock = cfg->gen_lock;  in xilinx_vdma_channel_set_config()
1046 	chan->config.master = cfg->master;  in xilinx_vdma_channel_set_config()
1048 	if (cfg->gen_lock && chan->genlock) {  in xilinx_vdma_channel_set_config()
1053 	chan->config.frm_cnt_en = cfg->frm_cnt_en;  in xilinx_vdma_channel_set_config()
1055 		chan->config.park_frm = cfg->park_frm;  in xilinx_vdma_channel_set_config()
1057 		chan->config.park_frm = -1;  in xilinx_vdma_channel_set_config()
1059 	chan->config.coalesc = cfg->coalesc;  in xilinx_vdma_channel_set_config()
1060 	chan->config.delay = cfg->delay;  in xilinx_vdma_channel_set_config()
1064 		chan->config.coalesc = cfg->coalesc;  in xilinx_vdma_channel_set_config()
1069 		chan->config.delay = cfg->delay;  in xilinx_vdma_channel_set_config()
1076 	vdma_ctrl_write(chan, XILINX_VDMA_REG_DMACR, dmacr);  in xilinx_vdma_channel_set_config()
1090 static void xilinx_vdma_chan_remove(struct xilinx_vdma_chan *chan)  in xilinx_vdma_chan_remove()  argument
1093 	vdma_ctrl_clr(chan, XILINX_VDMA_REG_DMACR,  in xilinx_vdma_chan_remove()
1096 	if (chan->irq > 0)  in xilinx_vdma_chan_remove()
1097 		free_irq(chan->irq, chan);  in xilinx_vdma_chan_remove()
1099 	tasklet_kill(&chan->tasklet);  in xilinx_vdma_chan_remove()
1101 	list_del(&chan->common.device_node);  in xilinx_vdma_chan_remove()
1117 	struct xilinx_vdma_chan *chan;  in xilinx_vdma_chan_probe()  local
1123 	chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL);  in xilinx_vdma_chan_probe()
1124 	if (!chan)  in xilinx_vdma_chan_probe()
1127 	chan->dev = xdev->dev;  in xilinx_vdma_chan_probe()
1128 	chan->xdev = xdev;  in xilinx_vdma_chan_probe()
1129 	chan->has_sg = xdev->has_sg;  in xilinx_vdma_chan_probe()
1131 	spin_lock_init(&chan->lock);  in xilinx_vdma_chan_probe()
1132 	INIT_LIST_HEAD(&chan->pending_list);  in xilinx_vdma_chan_probe()
1133 	INIT_LIST_HEAD(&chan->done_list);  in xilinx_vdma_chan_probe()
1138 	chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode");  in xilinx_vdma_chan_probe()
1155 		chan->direction = DMA_MEM_TO_DEV;  in xilinx_vdma_chan_probe()
1156 		chan->id = 0;  in xilinx_vdma_chan_probe()
1158 		chan->ctrl_offset = XILINX_VDMA_MM2S_CTRL_OFFSET;  in xilinx_vdma_chan_probe()
1159 		chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET;  in xilinx_vdma_chan_probe()
1163 			chan->flush_on_fsync = true;  in xilinx_vdma_chan_probe()
1166 		chan->direction = DMA_DEV_TO_MEM;  in xilinx_vdma_chan_probe()
1167 		chan->id = 1;  in xilinx_vdma_chan_probe()
1169 		chan->ctrl_offset = XILINX_VDMA_S2MM_CTRL_OFFSET;  in xilinx_vdma_chan_probe()
1170 		chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET;  in xilinx_vdma_chan_probe()
1174 			chan->flush_on_fsync = true;  in xilinx_vdma_chan_probe()
1181 	chan->irq = irq_of_parse_and_map(node, 0);  in xilinx_vdma_chan_probe()
1182 	err = request_irq(chan->irq, xilinx_vdma_irq_handler, IRQF_SHARED,  in xilinx_vdma_chan_probe()
1183 			  "xilinx-vdma-controller", chan);  in xilinx_vdma_chan_probe()
1185 		dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq);  in xilinx_vdma_chan_probe()
1190 	tasklet_init(&chan->tasklet, xilinx_vdma_do_tasklet,  in xilinx_vdma_chan_probe()
1191 			(unsigned long)chan);  in xilinx_vdma_chan_probe()
1197 	chan->common.device = &xdev->common;  in xilinx_vdma_chan_probe()
1199 	list_add_tail(&chan->common.device_node, &xdev->common.channels);  in xilinx_vdma_chan_probe()
1200 	xdev->chan[chan->id] = chan;  in xilinx_vdma_chan_probe()
1203 	err = xilinx_vdma_chan_reset(chan);  in xilinx_vdma_chan_probe()
1228 	return dma_get_slave_channel(&xdev->chan[chan_id]->common);  in of_dma_xilinx_xlate()
1300 		if (xdev->chan[i])  in xilinx_vdma_probe()
1301 			xdev->chan[i]->num_frms = num_frames;  in xilinx_vdma_probe()
1320 		if (xdev->chan[i])  in xilinx_vdma_probe()
1321 			xilinx_vdma_chan_remove(xdev->chan[i]);  in xilinx_vdma_probe()
1342 		if (xdev->chan[i])  in xilinx_vdma_remove()
1343 			xilinx_vdma_chan_remove(xdev->chan[i]);  in xilinx_vdma_remove()