655 	struct xgene_dma_ring *ring = &chan->tx_ring;  in xgene_chan_xfer_request()  local
659 	desc_hw = &ring->desc_hw[ring->head];  in xgene_chan_xfer_request()
665 	if (++ring->head == ring->slots)  in xgene_chan_xfer_request()
666 		ring->head = 0;  in xgene_chan_xfer_request()
676 		desc_hw = &ring->desc_hw[ring->head];  in xgene_chan_xfer_request()
678 		if (++ring->head == ring->slots)  in xgene_chan_xfer_request()
679 			ring->head = 0;  in xgene_chan_xfer_request()
690 		  2 : 1, ring->cmd);  in xgene_chan_xfer_request()
747 	struct xgene_dma_ring *ring = &chan->rx_ring;  in xgene_dma_cleanup_descriptors()  local
763 		desc_hw = &ring->desc_hw[ring->head];  in xgene_dma_cleanup_descriptors()
770 		if (++ring->head == ring->slots)  in xgene_dma_cleanup_descriptors()
771 			ring->head = 0;  in xgene_dma_cleanup_descriptors()
798 		iowrite32(-1, ring->cmd);  in xgene_dma_cleanup_descriptors()
1198 static void xgene_dma_wr_ring_state(struct xgene_dma_ring *ring)  in xgene_dma_wr_ring_state()  argument
1202 	iowrite32(ring->num, ring->pdma->csr_ring + XGENE_DMA_RING_STATE);  in xgene_dma_wr_ring_state()
1205 		iowrite32(ring->state[i], ring->pdma->csr_ring +  in xgene_dma_wr_ring_state()
1209 static void xgene_dma_clr_ring_state(struct xgene_dma_ring *ring)  in xgene_dma_clr_ring_state()  argument
1211 	memset(ring->state, 0, sizeof(u32) * XGENE_DMA_RING_NUM_CONFIG);  in xgene_dma_clr_ring_state()
1212 	xgene_dma_wr_ring_state(ring);  in xgene_dma_clr_ring_state()
1215 static void xgene_dma_setup_ring(struct xgene_dma_ring *ring)  in xgene_dma_setup_ring()  argument
1217 	void *ring_cfg = ring->state;  in xgene_dma_setup_ring()
1218 	u64 addr = ring->desc_paddr;  in xgene_dma_setup_ring()
1221 	ring->slots = ring->size / XGENE_DMA_RING_WQ_DESC_SIZE;  in xgene_dma_setup_ring()
1224 	xgene_dma_clr_ring_state(ring);  in xgene_dma_setup_ring()
1229 	if (ring->owner == XGENE_DMA_RING_OWNER_DMA) {  in xgene_dma_setup_ring()
1242 	XGENE_DMA_RING_SIZE_SET(ring_cfg, ring->cfgsize);  in xgene_dma_setup_ring()
1245 	xgene_dma_wr_ring_state(ring);  in xgene_dma_setup_ring()
1248 	iowrite32(XGENE_DMA_RING_ID_SETUP(ring->id),  in xgene_dma_setup_ring()
1249 		  ring->pdma->csr_ring + XGENE_DMA_RING_ID);  in xgene_dma_setup_ring()
1252 	iowrite32(XGENE_DMA_RING_ID_BUF_SETUP(ring->num),  in xgene_dma_setup_ring()
1253 		  ring->pdma->csr_ring + XGENE_DMA_RING_ID_BUF);  in xgene_dma_setup_ring()
1255 	if (ring->owner != XGENE_DMA_RING_OWNER_CPU)  in xgene_dma_setup_ring()
1259 	for (i = 0; i < ring->slots; i++) {  in xgene_dma_setup_ring()
1262 		desc = &ring->desc_hw[i];  in xgene_dma_setup_ring()
1267 	val = ioread32(ring->pdma->csr_ring + XGENE_DMA_RING_NE_INT_MODE);  in xgene_dma_setup_ring()
1268 	XGENE_DMA_RING_NE_INT_MODE_SET(val, ring->buf_num);  in xgene_dma_setup_ring()
1269 	iowrite32(val, ring->pdma->csr_ring + XGENE_DMA_RING_NE_INT_MODE);  in xgene_dma_setup_ring()
1272 static void xgene_dma_clear_ring(struct xgene_dma_ring *ring)  in xgene_dma_clear_ring()  argument
1276 	if (ring->owner == XGENE_DMA_RING_OWNER_CPU) {  in xgene_dma_clear_ring()
1278 		val = ioread32(ring->pdma->csr_ring +  in xgene_dma_clear_ring()
1280 		XGENE_DMA_RING_NE_INT_MODE_RESET(val, ring->buf_num);  in xgene_dma_clear_ring()
1281 		iowrite32(val, ring->pdma->csr_ring +  in xgene_dma_clear_ring()
1286 	ring_id = XGENE_DMA_RING_ID_SETUP(ring->id);  in xgene_dma_clear_ring()
1287 	iowrite32(ring_id, ring->pdma->csr_ring + XGENE_DMA_RING_ID);  in xgene_dma_clear_ring()
1289 	iowrite32(0, ring->pdma->csr_ring + XGENE_DMA_RING_ID_BUF);  in xgene_dma_clear_ring()
1290 	xgene_dma_clr_ring_state(ring);  in xgene_dma_clear_ring()
1293 static void xgene_dma_set_ring_cmd(struct xgene_dma_ring *ring)  in xgene_dma_set_ring_cmd()  argument
1295 	ring->cmd_base = ring->pdma->csr_ring_cmd +  in xgene_dma_set_ring_cmd()
1296 				XGENE_DMA_RING_CMD_BASE_OFFSET((ring->num -  in xgene_dma_set_ring_cmd()
1299 	ring->cmd = ring->cmd_base + XGENE_DMA_RING_CMD_OFFSET;  in xgene_dma_set_ring_cmd()
1331 static void xgene_dma_delete_ring_one(struct xgene_dma_ring *ring)  in xgene_dma_delete_ring_one()  argument
1334 	xgene_dma_clear_ring(ring);  in xgene_dma_delete_ring_one()
1337 	if (ring->desc_vaddr) {  in xgene_dma_delete_ring_one()
1338 		dma_free_coherent(ring->pdma->dev, ring->size,  in xgene_dma_delete_ring_one()
1339 				  ring->desc_vaddr, ring->desc_paddr);  in xgene_dma_delete_ring_one()
1340 		ring->desc_vaddr = NULL;  in xgene_dma_delete_ring_one()
1351 				     struct xgene_dma_ring *ring,  in xgene_dma_create_ring_one()  argument
1357 	ring->pdma = chan->pdma;  in xgene_dma_create_ring_one()
1358 	ring->cfgsize = cfgsize;  in xgene_dma_create_ring_one()
1359 	ring->num = chan->pdma->ring_num++;  in xgene_dma_create_ring_one()
1360 	ring->id = XGENE_DMA_RING_ID_GET(ring->owner, ring->buf_num);  in xgene_dma_create_ring_one()
1365 	ring->size = ret;  in xgene_dma_create_ring_one()
1368 	ring->desc_vaddr = dma_zalloc_coherent(chan->dev, ring->size,  in xgene_dma_create_ring_one()
1369 					       &ring->desc_paddr, GFP_KERNEL);  in xgene_dma_create_ring_one()
1370 	if (!ring->desc_vaddr) {  in xgene_dma_create_ring_one()
1376 	xgene_dma_set_ring_cmd(ring);  in xgene_dma_create_ring_one()
1377 	xgene_dma_setup_ring(ring);  in xgene_dma_create_ring_one()