205 #define chan_dbg(chan, fmt, arg...)	\  argument
206 	dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)
207 #define chan_err(chan, fmt, arg...)	\  argument
208 	dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
321 	struct xgene_dma_chan chan[XGENE_DMA_MAX_CHANNEL];  member
428 static void xgene_dma_prep_cpy_desc(struct xgene_dma_chan *chan,  in xgene_dma_prep_cpy_desc()  argument
438 	xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num);  in xgene_dma_prep_cpy_desc()
470 static void xgene_dma_prep_xor_desc(struct xgene_dma_chan *chan,  in xgene_dma_prep_xor_desc()  argument
484 	xgene_dma_init_desc(desc1, chan->tx_ring.dst_ring_num);  in xgene_dma_prep_xor_desc()
516 	struct xgene_dma_chan *chan;  in xgene_dma_tx_submit()  local
522 	chan = to_dma_chan(tx->chan);  in xgene_dma_tx_submit()
525 	spin_lock_bh(&chan->lock);  in xgene_dma_tx_submit()
530 	list_splice_tail_init(&desc->tx_list, &chan->ld_pending);  in xgene_dma_tx_submit()
532 	spin_unlock_bh(&chan->lock);  in xgene_dma_tx_submit()
537 static void xgene_dma_clean_descriptor(struct xgene_dma_chan *chan,  in xgene_dma_clean_descriptor()  argument
541 	chan_dbg(chan, "LD %p free\n", desc);  in xgene_dma_clean_descriptor()
542 	dma_pool_free(chan->desc_pool, desc, desc->tx.phys);  in xgene_dma_clean_descriptor()
546 				 struct xgene_dma_chan *chan)  in xgene_dma_alloc_descriptor()  argument
551 	desc = dma_pool_zalloc(chan->desc_pool, GFP_NOWAIT, &phys);  in xgene_dma_alloc_descriptor()
553 		chan_err(chan, "Failed to allocate LDs\n");  in xgene_dma_alloc_descriptor()
560 	dma_async_tx_descriptor_init(&desc->tx, &chan->dma_chan);  in xgene_dma_alloc_descriptor()
562 	chan_dbg(chan, "LD %p allocated\n", desc);  in xgene_dma_alloc_descriptor()
574 static void xgene_dma_clean_completed_descriptor(struct xgene_dma_chan *chan)  in xgene_dma_clean_completed_descriptor()  argument
579 	list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node) {  in xgene_dma_clean_completed_descriptor()
581 			xgene_dma_clean_descriptor(chan, desc);  in xgene_dma_clean_completed_descriptor()
593 static void xgene_dma_run_tx_complete_actions(struct xgene_dma_chan *chan,  in xgene_dma_run_tx_complete_actions()  argument
629 static void xgene_dma_clean_running_descriptor(struct xgene_dma_chan *chan,  in xgene_dma_clean_running_descriptor()  argument
644 		list_add_tail(&desc->node, &chan->ld_completed);  in xgene_dma_clean_running_descriptor()
648 	chan_dbg(chan, "LD %p free\n", desc);  in xgene_dma_clean_running_descriptor()
649 	dma_pool_free(chan->desc_pool, desc, desc->tx.phys);  in xgene_dma_clean_running_descriptor()
652 static void xgene_chan_xfer_request(struct xgene_dma_chan *chan,  in xgene_chan_xfer_request()  argument
655 	struct xgene_dma_ring *ring = &chan->tx_ring;  in xgene_chan_xfer_request()
685 	chan->pending += ((desc_sw->flags &  in xgene_chan_xfer_request()
699 static void xgene_chan_xfer_ld_pending(struct xgene_dma_chan *chan)  in xgene_chan_xfer_ld_pending()  argument
707 	if (list_empty(&chan->ld_pending)) {  in xgene_chan_xfer_ld_pending()
708 		chan_dbg(chan, "No pending LDs\n");  in xgene_chan_xfer_ld_pending()
716 	list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_pending, node) {  in xgene_chan_xfer_ld_pending()
723 		if (chan->pending >= chan->max_outstanding)  in xgene_chan_xfer_ld_pending()
726 		xgene_chan_xfer_request(chan, desc_sw);  in xgene_chan_xfer_ld_pending()
732 		list_move_tail(&desc_sw->node, &chan->ld_running);  in xgene_chan_xfer_ld_pending()
745 static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan)  in xgene_dma_cleanup_descriptors()  argument
747 	struct xgene_dma_ring *ring = &chan->rx_ring;  in xgene_dma_cleanup_descriptors()
755 	spin_lock_bh(&chan->lock);  in xgene_dma_cleanup_descriptors()
758 	xgene_dma_clean_completed_descriptor(chan);  in xgene_dma_cleanup_descriptors()
761 	list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_running, node) {  in xgene_dma_cleanup_descriptors()
781 			chan_err(chan, "%s\n", xgene_dma_desc_err[status]);  in xgene_dma_cleanup_descriptors()
807 		chan->pending -= ((desc_sw->flags &  in xgene_dma_cleanup_descriptors()
822 	xgene_chan_xfer_ld_pending(chan);  in xgene_dma_cleanup_descriptors()
824 	spin_unlock_bh(&chan->lock);  in xgene_dma_cleanup_descriptors()
828 		xgene_dma_run_tx_complete_actions(chan, desc_sw);  in xgene_dma_cleanup_descriptors()
829 		xgene_dma_clean_running_descriptor(chan, desc_sw);  in xgene_dma_cleanup_descriptors()
835 	struct xgene_dma_chan *chan = to_dma_chan(dchan);  in xgene_dma_alloc_chan_resources()  local
838 	if (chan->desc_pool)  in xgene_dma_alloc_chan_resources()
841 	chan->desc_pool = dma_pool_create(chan->name, chan->dev,  in xgene_dma_alloc_chan_resources()
844 	if (!chan->desc_pool) {  in xgene_dma_alloc_chan_resources()
845 		chan_err(chan, "Failed to allocate descriptor pool\n");  in xgene_dma_alloc_chan_resources()
849 	chan_dbg(chan, "Allocate descripto pool\n");  in xgene_dma_alloc_chan_resources()
861 static void xgene_dma_free_desc_list(struct xgene_dma_chan *chan,  in xgene_dma_free_desc_list()  argument
867 		xgene_dma_clean_descriptor(chan, desc);  in xgene_dma_free_desc_list()
872 	struct xgene_dma_chan *chan = to_dma_chan(dchan);  in xgene_dma_free_chan_resources()  local
874 	chan_dbg(chan, "Free all resources\n");  in xgene_dma_free_chan_resources()
876 	if (!chan->desc_pool)  in xgene_dma_free_chan_resources()
880 	xgene_dma_cleanup_descriptors(chan);  in xgene_dma_free_chan_resources()
882 	spin_lock_bh(&chan->lock);  in xgene_dma_free_chan_resources()
885 	xgene_dma_free_desc_list(chan, &chan->ld_pending);  in xgene_dma_free_chan_resources()
886 	xgene_dma_free_desc_list(chan, &chan->ld_running);  in xgene_dma_free_chan_resources()
887 	xgene_dma_free_desc_list(chan, &chan->ld_completed);  in xgene_dma_free_chan_resources()
889 	spin_unlock_bh(&chan->lock);  in xgene_dma_free_chan_resources()
892 	dma_pool_destroy(chan->desc_pool);  in xgene_dma_free_chan_resources()
893 	chan->desc_pool = NULL;  in xgene_dma_free_chan_resources()
902 	struct xgene_dma_chan *chan;  in xgene_dma_prep_sg()  local
916 	chan = to_dma_chan(dchan);  in xgene_dma_prep_sg()
936 		new = xgene_dma_alloc_descriptor(chan);  in xgene_dma_prep_sg()
941 		xgene_dma_prep_cpy_desc(chan, new, dst, src, len);  in xgene_dma_prep_sg()
1000 	xgene_dma_free_desc_list(chan, &first->tx_list);  in xgene_dma_prep_sg()
1009 	struct xgene_dma_chan *chan;  in xgene_dma_prep_xor()  local
1016 	chan = to_dma_chan(dchan);  in xgene_dma_prep_xor()
1020 		new = xgene_dma_alloc_descriptor(chan);  in xgene_dma_prep_xor()
1025 		xgene_dma_prep_xor_desc(chan, new, &dst, src,  in xgene_dma_prep_xor()
1048 	xgene_dma_free_desc_list(chan, &first->tx_list);  in xgene_dma_prep_xor()
1057 	struct xgene_dma_chan *chan;  in xgene_dma_prep_pq()  local
1065 	chan = to_dma_chan(dchan);  in xgene_dma_prep_pq()
1082 		new = xgene_dma_alloc_descriptor(chan);  in xgene_dma_prep_pq()
1100 			xgene_dma_prep_xor_desc(chan, new, &dst[0], src,  in xgene_dma_prep_pq()
1110 			xgene_dma_prep_xor_desc(chan, new, &dst[1], _src,  in xgene_dma_prep_pq()
1125 	xgene_dma_free_desc_list(chan, &first->tx_list);  in xgene_dma_prep_pq()
1131 	struct xgene_dma_chan *chan = to_dma_chan(dchan);  in xgene_dma_issue_pending()  local
1133 	spin_lock_bh(&chan->lock);  in xgene_dma_issue_pending()
1134 	xgene_chan_xfer_ld_pending(chan);  in xgene_dma_issue_pending()
1135 	spin_unlock_bh(&chan->lock);  in xgene_dma_issue_pending()
1147 	struct xgene_dma_chan *chan = (struct xgene_dma_chan *)data;  in xgene_dma_tasklet_cb()  local
1150 	xgene_dma_cleanup_descriptors(chan);  in xgene_dma_tasklet_cb()
1153 	enable_irq(chan->rx_irq);  in xgene_dma_tasklet_cb()
1158 	struct xgene_dma_chan *chan = (struct xgene_dma_chan *)id;  in xgene_dma_chan_ring_isr()  local
1160 	BUG_ON(!chan);  in xgene_dma_chan_ring_isr()
1166 	disable_irq_nosync(chan->rx_irq);  in xgene_dma_chan_ring_isr()
1173 	tasklet_schedule(&chan->tasklet);  in xgene_dma_chan_ring_isr()
1302 static int xgene_dma_get_ring_size(struct xgene_dma_chan *chan,  in xgene_dma_get_ring_size()  argument
1324 		chan_err(chan, "Unsupported cfg ring size %d\n", cfgsize);  in xgene_dma_get_ring_size()
1344 static void xgene_dma_delete_chan_rings(struct xgene_dma_chan *chan)  in xgene_dma_delete_chan_rings()  argument
1346 	xgene_dma_delete_ring_one(&chan->rx_ring);  in xgene_dma_delete_chan_rings()
1347 	xgene_dma_delete_ring_one(&chan->tx_ring);  in xgene_dma_delete_chan_rings()
1350 static int xgene_dma_create_ring_one(struct xgene_dma_chan *chan,  in xgene_dma_create_ring_one()  argument
1357 	ring->pdma = chan->pdma;  in xgene_dma_create_ring_one()
1359 	ring->num = chan->pdma->ring_num++;  in xgene_dma_create_ring_one()
1362 	ret = xgene_dma_get_ring_size(chan, cfgsize);  in xgene_dma_create_ring_one()
1368 	ring->desc_vaddr = dma_zalloc_coherent(chan->dev, ring->size,  in xgene_dma_create_ring_one()
1371 		chan_err(chan, "Failed to allocate ring desc\n");  in xgene_dma_create_ring_one()
1382 static int xgene_dma_create_chan_rings(struct xgene_dma_chan *chan)  in xgene_dma_create_chan_rings()  argument
1384 	struct xgene_dma_ring *rx_ring = &chan->rx_ring;  in xgene_dma_create_chan_rings()
1385 	struct xgene_dma_ring *tx_ring = &chan->tx_ring;  in xgene_dma_create_chan_rings()
1390 	rx_ring->buf_num = XGENE_DMA_CPU_BUFNUM + chan->id;  in xgene_dma_create_chan_rings()
1392 	ret = xgene_dma_create_ring_one(chan, rx_ring,  in xgene_dma_create_chan_rings()
1397 	chan_dbg(chan, "Rx ring id 0x%X num %d desc 0x%p\n",  in xgene_dma_create_chan_rings()
1402 	tx_ring->buf_num = XGENE_DMA_BUFNUM + chan->id;  in xgene_dma_create_chan_rings()
1404 	ret = xgene_dma_create_ring_one(chan, tx_ring,  in xgene_dma_create_chan_rings()
1413 	chan_dbg(chan,  in xgene_dma_create_chan_rings()
1418 	chan->max_outstanding = tx_ring->slots;  in xgene_dma_create_chan_rings()
1428 		ret = xgene_dma_create_chan_rings(&pdma->chan[i]);  in xgene_dma_init_rings()
1431 				xgene_dma_delete_chan_rings(&pdma->chan[j]);  in xgene_dma_init_rings()
1599 	struct xgene_dma_chan *chan;  in xgene_dma_request_irqs()  local
1613 		chan = &pdma->chan[i];  in xgene_dma_request_irqs()
1614 		irq_set_status_flags(chan->rx_irq, IRQ_DISABLE_UNLAZY);  in xgene_dma_request_irqs()
1615 		ret = devm_request_irq(chan->dev, chan->rx_irq,  in xgene_dma_request_irqs()
1617 				       0, chan->name, chan);  in xgene_dma_request_irqs()
1619 			chan_err(chan, "Failed to register Rx IRQ %d\n",  in xgene_dma_request_irqs()
1620 				 chan->rx_irq);  in xgene_dma_request_irqs()
1624 				chan = &pdma->chan[i];  in xgene_dma_request_irqs()
1625 				irq_clear_status_flags(chan->rx_irq, IRQ_DISABLE_UNLAZY);  in xgene_dma_request_irqs()
1626 				devm_free_irq(chan->dev, chan->rx_irq, chan);  in xgene_dma_request_irqs()
1638 	struct xgene_dma_chan *chan;  in xgene_dma_free_irqs()  local
1645 		chan = &pdma->chan[i];  in xgene_dma_free_irqs()
1646 		irq_clear_status_flags(chan->rx_irq, IRQ_DISABLE_UNLAZY);  in xgene_dma_free_irqs()
1647 		devm_free_irq(chan->dev, chan->rx_irq, chan);  in xgene_dma_free_irqs()
1651 static void xgene_dma_set_caps(struct xgene_dma_chan *chan,  in xgene_dma_set_caps()  argument
1669 	if ((chan->id == XGENE_DMA_PQ_CHANNEL) &&  in xgene_dma_set_caps()
1670 	    is_pq_enabled(chan->pdma)) {  in xgene_dma_set_caps()
1673 	} else if ((chan->id == XGENE_DMA_XOR_CHANNEL) &&  in xgene_dma_set_caps()
1674 		   !is_pq_enabled(chan->pdma)) {  in xgene_dma_set_caps()
1679 	dma_dev->dev = chan->dev;  in xgene_dma_set_caps()
1701 	struct xgene_dma_chan *chan = &pdma->chan[id];  in xgene_dma_async_register()  local
1705 	chan->dma_chan.device = dma_dev;  in xgene_dma_async_register()
1707 	spin_lock_init(&chan->lock);  in xgene_dma_async_register()
1708 	INIT_LIST_HEAD(&chan->ld_pending);  in xgene_dma_async_register()
1709 	INIT_LIST_HEAD(&chan->ld_running);  in xgene_dma_async_register()
1710 	INIT_LIST_HEAD(&chan->ld_completed);  in xgene_dma_async_register()
1711 	tasklet_init(&chan->tasklet, xgene_dma_tasklet_cb,  in xgene_dma_async_register()
1712 		     (unsigned long)chan);  in xgene_dma_async_register()
1714 	chan->pending = 0;  in xgene_dma_async_register()
1715 	chan->desc_pool = NULL;  in xgene_dma_async_register()
1716 	dma_cookie_init(&chan->dma_chan);  in xgene_dma_async_register()
1719 	xgene_dma_set_caps(chan, dma_dev);  in xgene_dma_async_register()
1723 	list_add_tail(&chan->dma_chan.device_node, &dma_dev->channels);  in xgene_dma_async_register()
1728 		chan_err(chan, "Failed to register async device %d", ret);  in xgene_dma_async_register()
1729 		tasklet_kill(&chan->tasklet);  in xgene_dma_async_register()
1736 		 "%s: CAPABILITY ( %s%s%s)\n", dma_chan_name(&chan->dma_chan),  in xgene_dma_async_register()
1753 				tasklet_kill(&pdma->chan[j].tasklet);  in xgene_dma_init_async()
1773 	struct xgene_dma_chan *chan;  in xgene_dma_init_channels()  local
1779 		chan = &pdma->chan[i];  in xgene_dma_init_channels()
1780 		chan->dev = pdma->dev;  in xgene_dma_init_channels()
1781 		chan->pdma = pdma;  in xgene_dma_init_channels()
1782 		chan->id = i;  in xgene_dma_init_channels()
1783 		snprintf(chan->name, sizeof(chan->name), "dmachan%d", chan->id);  in xgene_dma_init_channels()
1868 		pdma->chan[i - 1].rx_irq = irq;  in xgene_dma_get_resources()
1943 		xgene_dma_delete_chan_rings(&pdma->chan[i]);  in xgene_dma_probe()
1956 	struct xgene_dma_chan *chan;  in xgene_dma_remove()  local
1967 		chan = &pdma->chan[i];  in xgene_dma_remove()
1968 		tasklet_kill(&chan->tasklet);  in xgene_dma_remove()
1969 		xgene_dma_delete_chan_rings(chan);  in xgene_dma_remove()