43 static int tsi721_submit_sg(struct tsi721_tx_desc *desc);
336 tsi721_desc_fill_init(struct tsi721_tx_desc *desc,  in tsi721_desc_fill_init()  argument
347 				      (desc->rtype << 19) | desc->destid);  in tsi721_desc_fill_init()
348 	bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) |  in tsi721_desc_fill_init()
350 	rio_addr = (desc->rio_addr >> 2) |  in tsi721_desc_fill_init()
351 				((u64)(desc->rio_addr_u & 0x3) << 62);  in tsi721_desc_fill_init()
378 			      struct tsi721_tx_desc *desc)  in tsi721_dma_tx_err()  argument
380 	struct dma_async_tx_descriptor *txd = &desc->txd;  in tsi721_dma_tx_err()
384 	list_move(&desc->desc_node, &bdma_chan->free_list);  in tsi721_dma_tx_err()
414 static int tsi721_submit_sg(struct tsi721_tx_desc *desc)  in tsi721_submit_sg()  argument
416 	struct dma_chan *dchan = desc->txd.chan;  in tsi721_submit_sg()
439 	rio_addr = desc->rio_addr;  in tsi721_submit_sg()
457 	for_each_sg(desc->sg, sg, desc->sg_len, i) {  in tsi721_submit_sg()
460 			i, desc->sg_len,  in tsi721_submit_sg()
487 		desc->rio_addr = rio_addr;  in tsi721_submit_sg()
493 			desc->sg = sg;  in tsi721_submit_sg()
494 			desc->sg_len -= i;  in tsi721_submit_sg()
499 		err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size);  in tsi721_submit_sg()
507 			bd_ptr, desc->destid, desc->rio_addr);  in tsi721_submit_sg()
524 			desc->sg_len = 0;  in tsi721_submit_sg()
539 	struct tsi721_tx_desc *desc;  in tsi721_advance_work()  local
553 		desc = tsi721_dma_first_active(bdma_chan);  in tsi721_advance_work()
554 		err = tsi721_submit_sg(desc);  in tsi721_advance_work()
558 			tsi721_dma_tx_err(bdma_chan, desc);  in tsi721_advance_work()
593 		struct tsi721_tx_desc *desc;  in tsi721_dma_tasklet()  local
597 		desc = tsi721_dma_first_active(bdma_chan);  in tsi721_dma_tasklet()
599 		if (desc->sg_len == 0) {  in tsi721_dma_tasklet()
603 			desc->status = DMA_COMPLETE;  in tsi721_dma_tasklet()
604 			dma_cookie_complete(&desc->txd);  in tsi721_dma_tasklet()
605 			if (desc->txd.flags & DMA_PREP_INTERRUPT) {  in tsi721_dma_tasklet()
606 				callback = desc->txd.callback;  in tsi721_dma_tasklet()
607 				param = desc->txd.callback_param;  in tsi721_dma_tasklet()
609 			list_move(&desc->desc_node, &bdma_chan->free_list);  in tsi721_dma_tasklet()
626 	struct tsi721_tx_desc *desc = to_tsi721_desc(txd);  in tsi721_tx_submit()  local
631 	if (!list_empty(&desc->desc_node)) {  in tsi721_tx_submit()
645 	desc->status = DMA_IN_PROGRESS;  in tsi721_tx_submit()
646 	list_add_tail(&desc->desc_node, &bdma_chan->queue);  in tsi721_tx_submit()
655 	struct tsi721_tx_desc *desc = NULL;  in tsi721_alloc_chan_resources()  local
672 	desc = kcalloc(TSI721_DMA_TX_QUEUE_SZ, sizeof(struct tsi721_tx_desc),  in tsi721_alloc_chan_resources()
674 	if (!desc) {  in tsi721_alloc_chan_resources()
681 	bdma_chan->tx_desc = desc;  in tsi721_alloc_chan_resources()
684 		dma_async_tx_descriptor_init(&desc[i].txd, dchan);  in tsi721_alloc_chan_resources()
685 		desc[i].txd.tx_submit = tsi721_tx_submit;  in tsi721_alloc_chan_resources()
686 		desc[i].txd.flags = DMA_CTRL_ACK;  in tsi721_alloc_chan_resources()
687 		list_add(&desc[i].desc_node, &bdma_chan->free_list);  in tsi721_alloc_chan_resources()
762 	struct tsi721_tx_desc *desc, *_d;  in tsi721_prep_rio_sg()  local
798 	list_for_each_entry_safe(desc, _d, &bdma_chan->free_list, desc_node) {  in tsi721_prep_rio_sg()
799 		if (async_tx_test_ack(&desc->txd)) {  in tsi721_prep_rio_sg()
800 			list_del_init(&desc->desc_node);  in tsi721_prep_rio_sg()
801 			desc->destid = rext->destid;  in tsi721_prep_rio_sg()
802 			desc->rio_addr = rext->rio_addr;  in tsi721_prep_rio_sg()
803 			desc->rio_addr_u = 0;  in tsi721_prep_rio_sg()
804 			desc->rtype = rtype;  in tsi721_prep_rio_sg()
805 			desc->sg_len	= sg_len;  in tsi721_prep_rio_sg()
806 			desc->sg	= sgl;  in tsi721_prep_rio_sg()
807 			txd		= &desc->txd;  in tsi721_prep_rio_sg()
821 	struct tsi721_tx_desc *desc, *_d;  in tsi721_terminate_all()  local
845 	list_for_each_entry_safe(desc, _d, &list, desc_node)  in tsi721_terminate_all()
846 		tsi721_dma_tx_err(bdma_chan, desc);  in tsi721_terminate_all()