419 static u32 xgene_dma_ring_desc_cnt(struct xgene_dma_ring *ring)  in xgene_dma_ring_desc_cnt()  argument
421 	u32 __iomem *cmd_base = ring->cmd_base;  in xgene_dma_ring_desc_cnt()
695 static int xgene_chan_xfer_request(struct xgene_dma_ring *ring,  in xgene_chan_xfer_request()  argument
701 	if (xgene_dma_ring_desc_cnt(ring) > (ring->slots - 2))  in xgene_chan_xfer_request()
705 	desc_hw = &ring->desc_hw[ring->head];  in xgene_chan_xfer_request()
711 	if (++ring->head == ring->slots)  in xgene_chan_xfer_request()
712 		ring->head = 0;  in xgene_chan_xfer_request()
722 		desc_hw = &ring->desc_hw[ring->head];  in xgene_chan_xfer_request()
724 		if (++ring->head == ring->slots)  in xgene_chan_xfer_request()
725 			ring->head = 0;  in xgene_chan_xfer_request()
732 		  2 : 1, ring->cmd);  in xgene_chan_xfer_request()
797 	struct xgene_dma_ring *ring = &chan->rx_ring;  in xgene_dma_cleanup_descriptors()  local
808 		desc_hw = &ring->desc_hw[ring->head];  in xgene_dma_cleanup_descriptors()
814 		if (++ring->head == ring->slots)  in xgene_dma_cleanup_descriptors()
815 			ring->head = 0;  in xgene_dma_cleanup_descriptors()
842 		iowrite32(-1, ring->cmd);  in xgene_dma_cleanup_descriptors()
1298 static void xgene_dma_wr_ring_state(struct xgene_dma_ring *ring)  in xgene_dma_wr_ring_state()  argument
1302 	iowrite32(ring->num, ring->pdma->csr_ring + XGENE_DMA_RING_STATE);  in xgene_dma_wr_ring_state()
1305 		iowrite32(ring->state[i], ring->pdma->csr_ring +  in xgene_dma_wr_ring_state()
1309 static void xgene_dma_clr_ring_state(struct xgene_dma_ring *ring)  in xgene_dma_clr_ring_state()  argument
1311 	memset(ring->state, 0, sizeof(u32) * XGENE_DMA_RING_NUM_CONFIG);  in xgene_dma_clr_ring_state()
1312 	xgene_dma_wr_ring_state(ring);  in xgene_dma_clr_ring_state()
1315 static void xgene_dma_setup_ring(struct xgene_dma_ring *ring)  in xgene_dma_setup_ring()  argument
1317 	void *ring_cfg = ring->state;  in xgene_dma_setup_ring()
1318 	u64 addr = ring->desc_paddr;  in xgene_dma_setup_ring()
1322 	ring->slots = ring->size / XGENE_DMA_RING_WQ_DESC_SIZE;  in xgene_dma_setup_ring()
1325 	xgene_dma_clr_ring_state(ring);  in xgene_dma_setup_ring()
1330 	if (ring->owner == XGENE_DMA_RING_OWNER_DMA) {  in xgene_dma_setup_ring()
1343 	XGENE_DMA_RING_SIZE_SET(ring_cfg, ring->cfgsize);  in xgene_dma_setup_ring()
1346 	xgene_dma_wr_ring_state(ring);  in xgene_dma_setup_ring()
1349 	iowrite32(XGENE_DMA_RING_ID_SETUP(ring->id),  in xgene_dma_setup_ring()
1350 		  ring->pdma->csr_ring + XGENE_DMA_RING_ID);  in xgene_dma_setup_ring()
1353 	iowrite32(XGENE_DMA_RING_ID_BUF_SETUP(ring->num),  in xgene_dma_setup_ring()
1354 		  ring->pdma->csr_ring + XGENE_DMA_RING_ID_BUF);  in xgene_dma_setup_ring()
1356 	if (ring->owner != XGENE_DMA_RING_OWNER_CPU)  in xgene_dma_setup_ring()
1360 	for (i = 0; i < ring->slots; i++) {  in xgene_dma_setup_ring()
1361 		desc = &ring->desc_hw[i];  in xgene_dma_setup_ring()
1366 	val = ioread32(ring->pdma->csr_ring + XGENE_DMA_RING_NE_INT_MODE);  in xgene_dma_setup_ring()
1367 	XGENE_DMA_RING_NE_INT_MODE_SET(val, ring->buf_num);  in xgene_dma_setup_ring()
1368 	iowrite32(val, ring->pdma->csr_ring + XGENE_DMA_RING_NE_INT_MODE);  in xgene_dma_setup_ring()
1371 static void xgene_dma_clear_ring(struct xgene_dma_ring *ring)  in xgene_dma_clear_ring()  argument
1375 	if (ring->owner == XGENE_DMA_RING_OWNER_CPU) {  in xgene_dma_clear_ring()
1377 		val = ioread32(ring->pdma->csr_ring +  in xgene_dma_clear_ring()
1379 		XGENE_DMA_RING_NE_INT_MODE_RESET(val, ring->buf_num);  in xgene_dma_clear_ring()
1380 		iowrite32(val, ring->pdma->csr_ring +  in xgene_dma_clear_ring()
1385 	ring_id = XGENE_DMA_RING_ID_SETUP(ring->id);  in xgene_dma_clear_ring()
1386 	iowrite32(ring_id, ring->pdma->csr_ring + XGENE_DMA_RING_ID);  in xgene_dma_clear_ring()
1388 	iowrite32(0, ring->pdma->csr_ring + XGENE_DMA_RING_ID_BUF);  in xgene_dma_clear_ring()
1389 	xgene_dma_clr_ring_state(ring);  in xgene_dma_clear_ring()
1392 static void xgene_dma_set_ring_cmd(struct xgene_dma_ring *ring)  in xgene_dma_set_ring_cmd()  argument
1394 	ring->cmd_base = ring->pdma->csr_ring_cmd +  in xgene_dma_set_ring_cmd()
1395 				XGENE_DMA_RING_CMD_BASE_OFFSET((ring->num -  in xgene_dma_set_ring_cmd()
1398 	ring->cmd = ring->cmd_base + XGENE_DMA_RING_CMD_OFFSET;  in xgene_dma_set_ring_cmd()
1430 static void xgene_dma_delete_ring_one(struct xgene_dma_ring *ring)  in xgene_dma_delete_ring_one()  argument
1433 	xgene_dma_clear_ring(ring);  in xgene_dma_delete_ring_one()
1436 	if (ring->desc_vaddr) {  in xgene_dma_delete_ring_one()
1437 		dma_free_coherent(ring->pdma->dev, ring->size,  in xgene_dma_delete_ring_one()
1438 				  ring->desc_vaddr, ring->desc_paddr);  in xgene_dma_delete_ring_one()
1439 		ring->desc_vaddr = NULL;  in xgene_dma_delete_ring_one()
1450 				     struct xgene_dma_ring *ring,  in xgene_dma_create_ring_one()  argument
1454 	ring->pdma = chan->pdma;  in xgene_dma_create_ring_one()
1455 	ring->cfgsize = cfgsize;  in xgene_dma_create_ring_one()
1456 	ring->num = chan->pdma->ring_num++;  in xgene_dma_create_ring_one()
1457 	ring->id = XGENE_DMA_RING_ID_GET(ring->owner, ring->buf_num);  in xgene_dma_create_ring_one()
1459 	ring->size = xgene_dma_get_ring_size(chan, cfgsize);  in xgene_dma_create_ring_one()
1460 	if (ring->size <= 0)  in xgene_dma_create_ring_one()
1461 		return ring->size;  in xgene_dma_create_ring_one()
1464 	ring->desc_vaddr = dma_zalloc_coherent(chan->dev, ring->size,  in xgene_dma_create_ring_one()
1465 					       &ring->desc_paddr, GFP_KERNEL);  in xgene_dma_create_ring_one()
1466 	if (!ring->desc_vaddr) {  in xgene_dma_create_ring_one()
1472 	xgene_dma_set_ring_cmd(ring);  in xgene_dma_create_ring_one()
1473 	xgene_dma_setup_ring(ring);  in xgene_dma_create_ring_one()