134 #define chan_read(chan, fld)		__raw_readl((chan)->fld)  argument
137 #define chan_write(chan, fld, v)	__raw_writel(v, (chan)->fld)  argument
140 #define cpdma_desc_to_port(chan, mode, directed)			\  argument
142 		if (!is_rx_chan(chan) && ((directed == 1) ||		\
497 	struct cpdma_chan *chan;  in cpdma_chan_create()  local
504 	chan = devm_kzalloc(ctlr->dev, sizeof(*chan), GFP_KERNEL);  in cpdma_chan_create()
505 	if (!chan)  in cpdma_chan_create()
511 		devm_kfree(ctlr->dev, chan);  in cpdma_chan_create()
515 	chan->ctlr	= ctlr;  in cpdma_chan_create()
516 	chan->state	= CPDMA_STATE_IDLE;  in cpdma_chan_create()
517 	chan->chan_num	= chan_num;  in cpdma_chan_create()
518 	chan->handler	= handler;  in cpdma_chan_create()
520 	if (is_rx_chan(chan)) {  in cpdma_chan_create()
521 		chan->hdp	= ctlr->params.rxhdp + offset;  in cpdma_chan_create()
522 		chan->cp	= ctlr->params.rxcp + offset;  in cpdma_chan_create()
523 		chan->rxfree	= ctlr->params.rxfree + offset;  in cpdma_chan_create()
524 		chan->int_set	= CPDMA_RXINTMASKSET;  in cpdma_chan_create()
525 		chan->int_clear	= CPDMA_RXINTMASKCLEAR;  in cpdma_chan_create()
526 		chan->td	= CPDMA_RXTEARDOWN;  in cpdma_chan_create()
527 		chan->dir	= DMA_FROM_DEVICE;  in cpdma_chan_create()
529 		chan->hdp	= ctlr->params.txhdp + offset;  in cpdma_chan_create()
530 		chan->cp	= ctlr->params.txcp + offset;  in cpdma_chan_create()
531 		chan->int_set	= CPDMA_TXINTMASKSET;  in cpdma_chan_create()
532 		chan->int_clear	= CPDMA_TXINTMASKCLEAR;  in cpdma_chan_create()
533 		chan->td	= CPDMA_TXTEARDOWN;  in cpdma_chan_create()
534 		chan->dir	= DMA_TO_DEVICE;  in cpdma_chan_create()
536 	chan->mask = BIT(chan_linear(chan));  in cpdma_chan_create()
538 	spin_lock_init(&chan->lock);  in cpdma_chan_create()
540 	ctlr->channels[chan_num] = chan;  in cpdma_chan_create()
542 	return chan;  in cpdma_chan_create()
546 int cpdma_chan_destroy(struct cpdma_chan *chan)  in cpdma_chan_destroy()  argument
551 	if (!chan)  in cpdma_chan_destroy()
553 	ctlr = chan->ctlr;  in cpdma_chan_destroy()
556 	if (chan->state != CPDMA_STATE_IDLE)  in cpdma_chan_destroy()
557 		cpdma_chan_stop(chan);  in cpdma_chan_destroy()
558 	ctlr->channels[chan->chan_num] = NULL;  in cpdma_chan_destroy()
564 int cpdma_chan_get_stats(struct cpdma_chan *chan,  in cpdma_chan_get_stats()  argument
568 	if (!chan)  in cpdma_chan_get_stats()
570 	spin_lock_irqsave(&chan->lock, flags);  in cpdma_chan_get_stats()
571 	memcpy(stats, &chan->stats, sizeof(*stats));  in cpdma_chan_get_stats()
572 	spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_get_stats()
577 int cpdma_chan_dump(struct cpdma_chan *chan)  in cpdma_chan_dump()  argument
580 	struct device *dev = chan->ctlr->dev;  in cpdma_chan_dump()
582 	spin_lock_irqsave(&chan->lock, flags);  in cpdma_chan_dump()
585 		 chan->chan_num, is_rx_chan(chan) ? "rx" : "tx",  in cpdma_chan_dump()
586 		 chan_linear(chan), cpdma_state_str[chan->state]);  in cpdma_chan_dump()
587 	dev_info(dev, "\thdp: %x\n", chan_read(chan, hdp));  in cpdma_chan_dump()
588 	dev_info(dev, "\tcp: %x\n", chan_read(chan, cp));  in cpdma_chan_dump()
589 	if (chan->rxfree) {  in cpdma_chan_dump()
591 			 chan_read(chan, rxfree));  in cpdma_chan_dump()
595 		 chan->stats.head_enqueue);  in cpdma_chan_dump()
597 		 chan->stats.tail_enqueue);  in cpdma_chan_dump()
599 		 chan->stats.pad_enqueue);  in cpdma_chan_dump()
601 		 chan->stats.misqueued);  in cpdma_chan_dump()
603 		 chan->stats.desc_alloc_fail);  in cpdma_chan_dump()
605 		 chan->stats.pad_alloc_fail);  in cpdma_chan_dump()
607 		 chan->stats.runt_receive_buff);  in cpdma_chan_dump()
609 		 chan->stats.runt_transmit_buff);  in cpdma_chan_dump()
611 		 chan->stats.empty_dequeue);  in cpdma_chan_dump()
613 		 chan->stats.busy_dequeue);  in cpdma_chan_dump()
615 		 chan->stats.good_dequeue);  in cpdma_chan_dump()
617 		 chan->stats.requeue);  in cpdma_chan_dump()
619 		 chan->stats.teardown_dequeue);  in cpdma_chan_dump()
621 	spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_dump()
625 static void __cpdma_chan_submit(struct cpdma_chan *chan,  in __cpdma_chan_submit()  argument
628 	struct cpdma_ctlr		*ctlr = chan->ctlr;  in __cpdma_chan_submit()
629 	struct cpdma_desc __iomem	*prev = chan->tail;  in __cpdma_chan_submit()
637 	if (!chan->head) {  in __cpdma_chan_submit()
638 		chan->stats.head_enqueue++;  in __cpdma_chan_submit()
639 		chan->head = desc;  in __cpdma_chan_submit()
640 		chan->tail = desc;  in __cpdma_chan_submit()
641 		if (chan->state == CPDMA_STATE_ACTIVE)  in __cpdma_chan_submit()
642 			chan_write(chan, hdp, desc_dma);  in __cpdma_chan_submit()
648 	chan->tail = desc;  in __cpdma_chan_submit()
649 	chan->stats.tail_enqueue++;  in __cpdma_chan_submit()
654 	    (chan->state == CPDMA_STATE_ACTIVE)) {  in __cpdma_chan_submit()
656 		chan_write(chan, hdp, desc_dma);  in __cpdma_chan_submit()
657 		chan->stats.misqueued++;  in __cpdma_chan_submit()
661 int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data,  in cpdma_chan_submit()  argument
664 	struct cpdma_ctlr		*ctlr = chan->ctlr;  in cpdma_chan_submit()
671 	spin_lock_irqsave(&chan->lock, flags);  in cpdma_chan_submit()
673 	if (chan->state == CPDMA_STATE_TEARDOWN) {  in cpdma_chan_submit()
678 	desc = cpdma_desc_alloc(ctlr->pool, 1, is_rx_chan(chan));  in cpdma_chan_submit()
680 		chan->stats.desc_alloc_fail++;  in cpdma_chan_submit()
687 		chan->stats.runt_transmit_buff++;  in cpdma_chan_submit()
690 	buffer = dma_map_single(ctlr->dev, data, len, chan->dir);  in cpdma_chan_submit()
699 	cpdma_desc_to_port(chan, mode, directed);  in cpdma_chan_submit()
709 	__cpdma_chan_submit(chan, desc);  in cpdma_chan_submit()
711 	if (chan->state == CPDMA_STATE_ACTIVE && chan->rxfree)  in cpdma_chan_submit()
712 		chan_write(chan, rxfree, 1);  in cpdma_chan_submit()
714 	chan->count++;  in cpdma_chan_submit()
717 	spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_submit()
722 bool cpdma_check_free_tx_desc(struct cpdma_chan *chan)  in cpdma_check_free_tx_desc()  argument
727 	struct cpdma_ctlr	*ctlr = chan->ctlr;  in cpdma_check_free_tx_desc()
745 static void __cpdma_chan_free(struct cpdma_chan *chan,  in __cpdma_chan_free()  argument
749 	struct cpdma_ctlr		*ctlr = chan->ctlr;  in __cpdma_chan_free()
759 	dma_unmap_single(ctlr->dev, buff_dma, origlen, chan->dir);  in __cpdma_chan_free()
761 	(*chan->handler)(token, outlen, status);  in __cpdma_chan_free()
764 static int __cpdma_chan_process(struct cpdma_chan *chan)  in __cpdma_chan_process()  argument
766 	struct cpdma_ctlr		*ctlr = chan->ctlr;  in __cpdma_chan_process()
774 	spin_lock_irqsave(&chan->lock, flags);  in __cpdma_chan_process()
776 	desc = chan->head;  in __cpdma_chan_process()
778 		chan->stats.empty_dequeue++;  in __cpdma_chan_process()
787 		chan->stats.busy_dequeue++;  in __cpdma_chan_process()
798 	chan->head = desc_from_phys(pool, desc_read(desc, hw_next));  in __cpdma_chan_process()
799 	chan_write(chan, cp, desc_dma);  in __cpdma_chan_process()
800 	chan->count--;  in __cpdma_chan_process()
801 	chan->stats.good_dequeue++;  in __cpdma_chan_process()
804 		chan->stats.requeue++;  in __cpdma_chan_process()
805 		chan_write(chan, hdp, desc_phys(pool, chan->head));  in __cpdma_chan_process()
808 	spin_unlock_irqrestore(&chan->lock, flags);  in __cpdma_chan_process()
814 	__cpdma_chan_free(chan, desc, outlen, cb_status);  in __cpdma_chan_process()
818 	spin_unlock_irqrestore(&chan->lock, flags);  in __cpdma_chan_process()
822 int cpdma_chan_process(struct cpdma_chan *chan, int quota)  in cpdma_chan_process()  argument
826 	if (chan->state != CPDMA_STATE_ACTIVE)  in cpdma_chan_process()
830 		ret = __cpdma_chan_process(chan);  in cpdma_chan_process()
839 int cpdma_chan_start(struct cpdma_chan *chan)  in cpdma_chan_start()  argument
841 	struct cpdma_ctlr	*ctlr = chan->ctlr;  in cpdma_chan_start()
845 	spin_lock_irqsave(&chan->lock, flags);  in cpdma_chan_start()
846 	if (chan->state != CPDMA_STATE_IDLE) {  in cpdma_chan_start()
847 		spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_start()
851 		spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_start()
854 	dma_reg_write(ctlr, chan->int_set, chan->mask);  in cpdma_chan_start()
855 	chan->state = CPDMA_STATE_ACTIVE;  in cpdma_chan_start()
856 	if (chan->head) {  in cpdma_chan_start()
857 		chan_write(chan, hdp, desc_phys(pool, chan->head));  in cpdma_chan_start()
858 		if (chan->rxfree)  in cpdma_chan_start()
859 			chan_write(chan, rxfree, chan->count);  in cpdma_chan_start()
862 	spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_start()
867 int cpdma_chan_stop(struct cpdma_chan *chan)  in cpdma_chan_stop()  argument
869 	struct cpdma_ctlr	*ctlr = chan->ctlr;  in cpdma_chan_stop()
875 	spin_lock_irqsave(&chan->lock, flags);  in cpdma_chan_stop()
876 	if (chan->state == CPDMA_STATE_TEARDOWN) {  in cpdma_chan_stop()
877 		spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_stop()
881 	chan->state = CPDMA_STATE_TEARDOWN;  in cpdma_chan_stop()
882 	dma_reg_write(ctlr, chan->int_clear, chan->mask);  in cpdma_chan_stop()
885 	dma_reg_write(ctlr, chan->td, chan_linear(chan));  in cpdma_chan_stop()
890 		u32 cp = chan_read(chan, cp);  in cpdma_chan_stop()
897 	chan_write(chan, cp, CPDMA_TEARDOWN_VALUE);  in cpdma_chan_stop()
900 	spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_stop()
902 		ret = __cpdma_chan_process(chan);  in cpdma_chan_stop()
906 	spin_lock_irqsave(&chan->lock, flags);  in cpdma_chan_stop()
909 	while (chan->head) {  in cpdma_chan_stop()
910 		struct cpdma_desc __iomem *desc = chan->head;  in cpdma_chan_stop()
914 		chan->head = desc_from_phys(pool, next_dma);  in cpdma_chan_stop()
915 		chan->count--;  in cpdma_chan_stop()
916 		chan->stats.teardown_dequeue++;  in cpdma_chan_stop()
919 		spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_stop()
920 		__cpdma_chan_free(chan, desc, 0, -ENOSYS);  in cpdma_chan_stop()
921 		spin_lock_irqsave(&chan->lock, flags);  in cpdma_chan_stop()
924 	chan->state = CPDMA_STATE_IDLE;  in cpdma_chan_stop()
925 	spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_stop()
930 int cpdma_chan_int_ctrl(struct cpdma_chan *chan, bool enable)  in cpdma_chan_int_ctrl()  argument
934 	spin_lock_irqsave(&chan->lock, flags);  in cpdma_chan_int_ctrl()
935 	if (chan->state != CPDMA_STATE_ACTIVE) {  in cpdma_chan_int_ctrl()
936 		spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_int_ctrl()
940 	dma_reg_write(chan->ctlr, enable ? chan->int_set : chan->int_clear,  in cpdma_chan_int_ctrl()
941 		      chan->mask);  in cpdma_chan_int_ctrl()
942 	spin_unlock_irqrestore(&chan->lock, flags);  in cpdma_chan_int_ctrl()