243 static inline bool imxdma_chan_is_doing_cyclic(struct imxdma_channel *imxdmac)  in imxdma_chan_is_doing_cyclic()  argument
247 	if (!list_empty(&imxdmac->ld_active)) {  in imxdma_chan_is_doing_cyclic()
248 		desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc,  in imxdma_chan_is_doing_cyclic()
269 static int imxdma_hw_chain(struct imxdma_channel *imxdmac)  in imxdma_hw_chain()  argument
271 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_hw_chain()
274 		return imxdmac->hw_chaining;  in imxdma_hw_chain()
284 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_sg_next()  local
285 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_sg_next()
295 				 DMA_DAR(imxdmac->channel));  in imxdma_sg_next()
298 				 DMA_SAR(imxdmac->channel));  in imxdma_sg_next()
300 	imx_dmav1_writel(imxdma, now, DMA_CNTR(imxdmac->channel));  in imxdma_sg_next()
303 		"size 0x%08x\n", __func__, imxdmac->channel,  in imxdma_sg_next()
304 		 imx_dmav1_readl(imxdma, DMA_DAR(imxdmac->channel)),  in imxdma_sg_next()
305 		 imx_dmav1_readl(imxdma, DMA_SAR(imxdmac->channel)),  in imxdma_sg_next()
306 		 imx_dmav1_readl(imxdma, DMA_CNTR(imxdmac->channel)));  in imxdma_sg_next()
313 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_enable_hw()  local
314 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_enable_hw()
315 	int channel = imxdmac->channel;  in imxdma_enable_hw()
329 			d->sg && imxdma_hw_chain(imxdmac)) {  in imxdma_enable_hw()
343 static void imxdma_disable_hw(struct imxdma_channel *imxdmac)  in imxdma_disable_hw()  argument
345 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_disable_hw()
346 	int channel = imxdmac->channel;  in imxdma_disable_hw()
351 	if (imxdma_hw_chain(imxdmac))  in imxdma_disable_hw()
352 		del_timer(&imxdmac->watchdog);  in imxdma_disable_hw()
365 	struct imxdma_channel *imxdmac = (struct imxdma_channel *)data;  in imxdma_watchdog()  local
366 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_watchdog()
367 	int channel = imxdmac->channel;  in imxdma_watchdog()
372 	tasklet_schedule(&imxdmac->dma_tasklet);  in imxdma_watchdog()
374 		imxdmac->channel);  in imxdma_watchdog()
430 static void dma_irq_handle_channel(struct imxdma_channel *imxdmac)  in dma_irq_handle_channel()  argument
432 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in dma_irq_handle_channel()
433 	int chno = imxdmac->channel;  in dma_irq_handle_channel()
438 	if (list_empty(&imxdmac->ld_active)) {  in dma_irq_handle_channel()
443 	desc = list_first_entry(&imxdmac->ld_active,  in dma_irq_handle_channel()
457 			if (imxdma_hw_chain(imxdmac)) {  in dma_irq_handle_channel()
461 				mod_timer(&imxdmac->watchdog,  in dma_irq_handle_channel()
474 			if (imxdma_chan_is_doing_cyclic(imxdmac))  in dma_irq_handle_channel()
476 				tasklet_schedule(&imxdmac->dma_tasklet);  in dma_irq_handle_channel()
481 		if (imxdma_hw_chain(imxdmac)) {  in dma_irq_handle_channel()
482 			del_timer(&imxdmac->watchdog);  in dma_irq_handle_channel()
490 	tasklet_schedule(&imxdmac->dma_tasklet);  in dma_irq_handle_channel()
516 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_xfer_desc()  local
517 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_xfer_desc()
542 		imxdmac->slot_2d = slot;  in imxdma_xfer_desc()
543 		imxdmac->enabled_2d = true;  in imxdma_xfer_desc()
563 		imx_dmav1_writel(imxdma, d->src, DMA_SAR(imxdmac->channel));  in imxdma_xfer_desc()
564 		imx_dmav1_writel(imxdma, d->dest, DMA_DAR(imxdmac->channel));  in imxdma_xfer_desc()
566 			 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
568 		imx_dmav1_writel(imxdma, d->len, DMA_CNTR(imxdmac->channel));  in imxdma_xfer_desc()
572 			__func__, imxdmac->channel,  in imxdma_xfer_desc()
581 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
582 					 DMA_SAR(imxdmac->channel));  in imxdma_xfer_desc()
583 			imx_dmav1_writel(imxdma, imxdmac->ccr_from_device,  in imxdma_xfer_desc()
584 					 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
588 				__func__, imxdmac->channel,  in imxdma_xfer_desc()
590 				(unsigned long long)imxdmac->per_address);  in imxdma_xfer_desc()
592 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
593 					 DMA_DAR(imxdmac->channel));  in imxdma_xfer_desc()
594 			imx_dmav1_writel(imxdma, imxdmac->ccr_to_device,  in imxdma_xfer_desc()
595 					 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
599 				__func__, imxdmac->channel,  in imxdma_xfer_desc()
601 				(unsigned long long)imxdmac->per_address);  in imxdma_xfer_desc()
604 				__func__, imxdmac->channel);  in imxdma_xfer_desc()
620 	struct imxdma_channel *imxdmac = (void *)data;  in imxdma_tasklet()  local
621 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tasklet()
627 	if (list_empty(&imxdmac->ld_active)) {  in imxdma_tasklet()
632 	desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node);  in imxdma_tasklet()
638 	if (imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_tasklet()
644 	if (imxdmac->enabled_2d) {  in imxdma_tasklet()
645 		imxdma->slots_2d[imxdmac->slot_2d].count--;  in imxdma_tasklet()
646 		imxdmac->enabled_2d = false;  in imxdma_tasklet()
649 	list_move_tail(imxdmac->ld_active.next, &imxdmac->ld_free);  in imxdma_tasklet()
651 	if (!list_empty(&imxdmac->ld_queue)) {  in imxdma_tasklet()
652 		desc = list_first_entry(&imxdmac->ld_queue, struct imxdma_desc,  in imxdma_tasklet()
654 		list_move_tail(imxdmac->ld_queue.next, &imxdmac->ld_active);  in imxdma_tasklet()
657 				 __func__, imxdmac->channel);  in imxdma_tasklet()
669 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_terminate_all()  local
670 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_terminate_all()
673 	imxdma_disable_hw(imxdmac);  in imxdma_terminate_all()
676 	list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);  in imxdma_terminate_all()
677 	list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);  in imxdma_terminate_all()
685 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_config()  local
686 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_config()
690 		imxdmac->per_address = dmaengine_cfg->src_addr;  in imxdma_config()
691 		imxdmac->watermark_level = dmaengine_cfg->src_maxburst;  in imxdma_config()
692 		imxdmac->word_size = dmaengine_cfg->src_addr_width;  in imxdma_config()
694 		imxdmac->per_address = dmaengine_cfg->dst_addr;  in imxdma_config()
695 		imxdmac->watermark_level = dmaengine_cfg->dst_maxburst;  in imxdma_config()
696 		imxdmac->word_size = dmaengine_cfg->dst_addr_width;  in imxdma_config()
699 	switch (imxdmac->word_size) {  in imxdma_config()
712 	imxdmac->hw_chaining = 0;  in imxdma_config()
714 	imxdmac->ccr_from_device = (mode | IMX_DMA_TYPE_FIFO) |  in imxdma_config()
717 	imxdmac->ccr_to_device =  in imxdma_config()
720 	imx_dmav1_writel(imxdma, imxdmac->dma_request,  in imxdma_config()
721 			 DMA_RSSR(imxdmac->channel));  in imxdma_config()
724 	imx_dmav1_writel(imxdma, imxdmac->watermark_level *  in imxdma_config()
725 			 imxdmac->word_size, DMA_BLR(imxdmac->channel));  in imxdma_config()
739 	struct imxdma_channel *imxdmac = to_imxdma_chan(tx->chan);  in imxdma_tx_submit()  local
740 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tx_submit()
745 	list_move_tail(imxdmac->ld_free.next, &imxdmac->ld_queue);  in imxdma_tx_submit()
754 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_alloc_chan_resources()  local
758 		imxdmac->dma_request = data->dma_request;  in imxdma_alloc_chan_resources()
760 	while (imxdmac->descs_allocated < IMXDMA_MAX_CHAN_DESCRIPTORS) {  in imxdma_alloc_chan_resources()
773 		list_add_tail(&desc->node, &imxdmac->ld_free);  in imxdma_alloc_chan_resources()
774 		imxdmac->descs_allocated++;  in imxdma_alloc_chan_resources()
777 	if (!imxdmac->descs_allocated)  in imxdma_alloc_chan_resources()
780 	return imxdmac->descs_allocated;  in imxdma_alloc_chan_resources()
785 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_free_chan_resources()  local
786 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_free_chan_resources()
792 	imxdma_disable_hw(imxdmac);  in imxdma_free_chan_resources()
793 	list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);  in imxdma_free_chan_resources()
794 	list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);  in imxdma_free_chan_resources()
798 	list_for_each_entry_safe(desc, _desc, &imxdmac->ld_free, node) {  in imxdma_free_chan_resources()
800 		imxdmac->descs_allocated--;  in imxdma_free_chan_resources()
802 	INIT_LIST_HEAD(&imxdmac->ld_free);  in imxdma_free_chan_resources()
804 	kfree(imxdmac->sg_list);  in imxdma_free_chan_resources()
805 	imxdmac->sg_list = NULL;  in imxdma_free_chan_resources()
813 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_slave_sg()  local
818 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_slave_sg()
819 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_slave_sg()
822 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_slave_sg()
828 	switch (imxdmac->word_size) {  in imxdma_prep_slave_sg()
849 		desc->src = imxdmac->per_address;  in imxdma_prep_slave_sg()
851 		desc->dest = imxdmac->per_address;  in imxdma_prep_slave_sg()
864 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_cyclic()  local
865 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_cyclic()
871 			__func__, imxdmac->channel, buf_len, period_len);  in imxdma_prep_dma_cyclic()
873 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_cyclic()
874 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_cyclic()
877 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_cyclic()
879 	kfree(imxdmac->sg_list);  in imxdma_prep_dma_cyclic()
881 	imxdmac->sg_list = kcalloc(periods + 1,  in imxdma_prep_dma_cyclic()
883 	if (!imxdmac->sg_list)  in imxdma_prep_dma_cyclic()
886 	sg_init_table(imxdmac->sg_list, periods);  in imxdma_prep_dma_cyclic()
889 		imxdmac->sg_list[i].page_link = 0;  in imxdma_prep_dma_cyclic()
890 		imxdmac->sg_list[i].offset = 0;  in imxdma_prep_dma_cyclic()
891 		imxdmac->sg_list[i].dma_address = dma_addr;  in imxdma_prep_dma_cyclic()
892 		sg_dma_len(&imxdmac->sg_list[i]) = period_len;  in imxdma_prep_dma_cyclic()
897 	imxdmac->sg_list[periods].offset = 0;  in imxdma_prep_dma_cyclic()
898 	sg_dma_len(&imxdmac->sg_list[periods]) = 0;  in imxdma_prep_dma_cyclic()
899 	imxdmac->sg_list[periods].page_link =  in imxdma_prep_dma_cyclic()
900 		((unsigned long)imxdmac->sg_list | 0x01) & ~0x02;  in imxdma_prep_dma_cyclic()
903 	desc->sg = imxdmac->sg_list;  in imxdma_prep_dma_cyclic()
908 		desc->src = imxdmac->per_address;  in imxdma_prep_dma_cyclic()
910 		desc->dest = imxdmac->per_address;  in imxdma_prep_dma_cyclic()
922 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_memcpy()  local
923 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_memcpy()
927 		__func__, imxdmac->channel, (unsigned long long)src,  in imxdma_prep_dma_memcpy()
930 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_memcpy()
931 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_memcpy()
934 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_memcpy()
953 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_interleaved()  local
954 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_interleaved()
959 		imxdmac->channel, (unsigned long long)xt->src_start,  in imxdma_prep_dma_interleaved()
964 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_interleaved()
965 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_interleaved()
971 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_interleaved()
995 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_issue_pending()  local
996 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_issue_pending()
1001 	if (list_empty(&imxdmac->ld_active) &&  in imxdma_issue_pending()
1002 	    !list_empty(&imxdmac->ld_queue)) {  in imxdma_issue_pending()
1003 		desc = list_first_entry(&imxdmac->ld_queue,  in imxdma_issue_pending()
1009 				 __func__, imxdmac->channel);  in imxdma_issue_pending()
1011 			list_move_tail(imxdmac->ld_queue.next,  in imxdma_issue_pending()
1012 				       &imxdmac->ld_active);  in imxdma_issue_pending()
1142 		struct imxdma_channel *imxdmac = &imxdma->channel[i];  in imxdma_probe()  local
1153 			init_timer(&imxdmac->watchdog);  in imxdma_probe()
1154 			imxdmac->watchdog.function = &imxdma_watchdog;  in imxdma_probe()
1155 			imxdmac->watchdog.data = (unsigned long)imxdmac;  in imxdma_probe()
1158 		imxdmac->imxdma = imxdma;  in imxdma_probe()
1160 		INIT_LIST_HEAD(&imxdmac->ld_queue);  in imxdma_probe()
1161 		INIT_LIST_HEAD(&imxdmac->ld_free);  in imxdma_probe()
1162 		INIT_LIST_HEAD(&imxdmac->ld_active);  in imxdma_probe()
1164 		tasklet_init(&imxdmac->dma_tasklet, imxdma_tasklet,  in imxdma_probe()
1165 			     (unsigned long)imxdmac);  in imxdma_probe()
1166 		imxdmac->chan.device = &imxdma->dma_device;  in imxdma_probe()
1167 		dma_cookie_init(&imxdmac->chan);  in imxdma_probe()
1168 		imxdmac->channel = i;  in imxdma_probe()
1171 		list_add_tail(&imxdmac->chan.device_node,  in imxdma_probe()