121 	struct axi_dmac_chan chan;  member
126 static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan)  in chan_to_axi_dmac()  argument
128 	return container_of(chan->vchan.chan.device, struct axi_dmac,  in chan_to_axi_dmac()
134 	return container_of(c, struct axi_dmac_chan, vchan.chan);  in to_axi_dmac_chan()
153 static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan)  in axi_dmac_src_is_mem()  argument
155 	return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM;  in axi_dmac_src_is_mem()
158 static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan)  in axi_dmac_dest_is_mem()  argument
160 	return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM;  in axi_dmac_dest_is_mem()
163 static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len)  in axi_dmac_check_len()  argument
165 	if (len == 0 || len > chan->max_length)  in axi_dmac_check_len()
167 	if ((len & chan->align_mask) != 0) /* Not aligned */  in axi_dmac_check_len()
172 static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr)  in axi_dmac_check_addr()  argument
174 	if ((addr & chan->align_mask) != 0) /* Not aligned */  in axi_dmac_check_addr()
179 static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)  in axi_dmac_start_transfer()  argument
181 	struct axi_dmac *dmac = chan_to_axi_dmac(chan);  in axi_dmac_start_transfer()
192 	desc = chan->next_desc;  in axi_dmac_start_transfer()
195 		vdesc = vchan_next_desc(&chan->vchan);  in axi_dmac_start_transfer()
198 		list_move_tail(&vdesc->node, &chan->active_descs);  in axi_dmac_start_transfer()
205 		chan->next_desc = NULL;  in axi_dmac_start_transfer()
207 		chan->next_desc = desc;  in axi_dmac_start_transfer()
211 	if (axi_dmac_dest_is_mem(chan)) {  in axi_dmac_start_transfer()
216 	if (axi_dmac_src_is_mem(chan)) {  in axi_dmac_start_transfer()
225 	if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback)  in axi_dmac_start_transfer()
234 static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan)  in axi_dmac_active_desc()  argument
236 	return list_first_entry_or_null(&chan->active_descs,  in axi_dmac_active_desc()
240 static void axi_dmac_transfer_done(struct axi_dmac_chan *chan,  in axi_dmac_transfer_done()  argument
246 	active = axi_dmac_active_desc(chan);  in axi_dmac_transfer_done()
261 				active = axi_dmac_active_desc(chan);  in axi_dmac_transfer_done()
275 	spin_lock(&dmac->chan.vchan.lock);  in axi_dmac_interrupt_handler()
281 		axi_dmac_transfer_done(&dmac->chan, completed);  in axi_dmac_interrupt_handler()
285 		axi_dmac_start_transfer(&dmac->chan);  in axi_dmac_interrupt_handler()
286 	spin_unlock(&dmac->chan.vchan.lock);  in axi_dmac_interrupt_handler()
293 	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);  in axi_dmac_terminate_all()  local
294 	struct axi_dmac *dmac = chan_to_axi_dmac(chan);  in axi_dmac_terminate_all()
298 	spin_lock_irqsave(&chan->vchan.lock, flags);  in axi_dmac_terminate_all()
300 	chan->next_desc = NULL;  in axi_dmac_terminate_all()
301 	vchan_get_all_descriptors(&chan->vchan, &head);  in axi_dmac_terminate_all()
302 	list_splice_tail_init(&chan->active_descs, &head);  in axi_dmac_terminate_all()
303 	spin_unlock_irqrestore(&chan->vchan.lock, flags);  in axi_dmac_terminate_all()
305 	vchan_dma_desc_free_list(&chan->vchan, &head);  in axi_dmac_terminate_all()
312 	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);  in axi_dmac_issue_pending()  local
313 	struct axi_dmac *dmac = chan_to_axi_dmac(chan);  in axi_dmac_issue_pending()
318 	spin_lock_irqsave(&chan->vchan.lock, flags);  in axi_dmac_issue_pending()
319 	if (vchan_issue_pending(&chan->vchan))  in axi_dmac_issue_pending()
320 		axi_dmac_start_transfer(chan);  in axi_dmac_issue_pending()
321 	spin_unlock_irqrestore(&chan->vchan.lock, flags);  in axi_dmac_issue_pending()
343 	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);  in axi_dmac_prep_slave_sg()  local
348 	if (direction != chan->direction)  in axi_dmac_prep_slave_sg()
356 		if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) ||  in axi_dmac_prep_slave_sg()
357 		    !axi_dmac_check_len(chan, sg_dma_len(sg))) {  in axi_dmac_prep_slave_sg()
372 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);  in axi_dmac_prep_slave_sg()
380 	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);  in axi_dmac_prep_dma_cyclic()  local
384 	if (direction != chan->direction)  in axi_dmac_prep_dma_cyclic()
387 	if (!axi_dmac_check_len(chan, buf_len) ||  in axi_dmac_prep_dma_cyclic()
388 	    !axi_dmac_check_addr(chan, buf_addr))  in axi_dmac_prep_dma_cyclic()
412 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);  in axi_dmac_prep_dma_cyclic()
419 	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);  in axi_dmac_prep_interleaved()  local
426 	if (xt->dir != chan->direction)  in axi_dmac_prep_interleaved()
429 	if (axi_dmac_src_is_mem(chan)) {  in axi_dmac_prep_interleaved()
430 		if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start))  in axi_dmac_prep_interleaved()
434 	if (axi_dmac_dest_is_mem(chan)) {  in axi_dmac_prep_interleaved()
435 		if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start))  in axi_dmac_prep_interleaved()
442 	if (chan->hw_2d) {  in axi_dmac_prep_interleaved()
443 		if (!axi_dmac_check_len(chan, xt->sgl[0].size) ||  in axi_dmac_prep_interleaved()
444 		    !axi_dmac_check_len(chan, xt->numf))  in axi_dmac_prep_interleaved()
446 		if (xt->sgl[0].size + dst_icg > chan->max_length ||  in axi_dmac_prep_interleaved()
447 		    xt->sgl[0].size + src_icg > chan->max_length)  in axi_dmac_prep_interleaved()
452 		if (chan->max_length / xt->sgl[0].size < xt->numf)  in axi_dmac_prep_interleaved()
454 		if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf))  in axi_dmac_prep_interleaved()
462 	if (axi_dmac_src_is_mem(chan)) {  in axi_dmac_prep_interleaved()
467 	if (axi_dmac_dest_is_mem(chan)) {  in axi_dmac_prep_interleaved()
472 	if (chan->hw_2d) {  in axi_dmac_prep_interleaved()
480 	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);  in axi_dmac_prep_interleaved()
499 	struct axi_dmac_chan *chan)  in axi_dmac_parse_chan_dt()  argument
517 	chan->src_type = val;  in axi_dmac_parse_chan_dt()
524 	chan->dest_type = val;  in axi_dmac_parse_chan_dt()
529 	chan->src_width = val / 8;  in axi_dmac_parse_chan_dt()
534 	chan->dest_width = val / 8;  in axi_dmac_parse_chan_dt()
541 		chan->max_length = UINT_MAX;  in axi_dmac_parse_chan_dt()
543 		chan->max_length = (1ULL << val) - 1;  in axi_dmac_parse_chan_dt()
545 	chan->align_mask = max(chan->dest_width, chan->src_width) - 1;  in axi_dmac_parse_chan_dt()
547 	if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))  in axi_dmac_parse_chan_dt()
548 		chan->direction = DMA_MEM_TO_MEM;  in axi_dmac_parse_chan_dt()
549 	else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))  in axi_dmac_parse_chan_dt()
550 		chan->direction = DMA_MEM_TO_DEV;  in axi_dmac_parse_chan_dt()
551 	else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan))  in axi_dmac_parse_chan_dt()
552 		chan->direction = DMA_DEV_TO_MEM;  in axi_dmac_parse_chan_dt()
554 		chan->direction = DMA_DEV_TO_DEV;  in axi_dmac_parse_chan_dt()
556 	chan->hw_cyclic = of_property_read_bool(of_chan, "adi,cyclic");  in axi_dmac_parse_chan_dt()
557 	chan->hw_2d = of_property_read_bool(of_chan, "adi,2d");  in axi_dmac_parse_chan_dt()
587 	INIT_LIST_HEAD(&dmac->chan.active_descs);  in axi_dmac_probe()
594 		ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan);  in axi_dmac_probe()
604 	dma_set_max_seg_size(&pdev->dev, dmac->chan.max_length);  in axi_dmac_probe()
618 	dma_dev->src_addr_widths = BIT(dmac->chan.src_width);  in axi_dmac_probe()
619 	dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width);  in axi_dmac_probe()
620 	dma_dev->directions = BIT(dmac->chan.direction);  in axi_dmac_probe()
624 	dmac->chan.vchan.desc_free = axi_dmac_desc_free;  in axi_dmac_probe()
625 	vchan_init(&dmac->chan.vchan, dma_dev);  in axi_dmac_probe()
667 	tasklet_kill(&dmac->chan.vchan.task);  in axi_dmac_remove()