107 	struct cpdma_desc_pool	*pool;  member
159 	struct cpdma_desc_pool *pool;  in cpdma_desc_pool_create()  local
161 	pool = devm_kzalloc(dev, sizeof(*pool), GFP_KERNEL);  in cpdma_desc_pool_create()
162 	if (!pool)  in cpdma_desc_pool_create()
165 	spin_lock_init(&pool->lock);  in cpdma_desc_pool_create()
167 	pool->dev	= dev;  in cpdma_desc_pool_create()
168 	pool->mem_size	= size;  in cpdma_desc_pool_create()
169 	pool->desc_size	= ALIGN(sizeof(struct cpdma_desc), align);  in cpdma_desc_pool_create()
170 	pool->num_desc	= size / pool->desc_size;  in cpdma_desc_pool_create()
172 	bitmap_size  = (pool->num_desc / BITS_PER_LONG) * sizeof(long);  in cpdma_desc_pool_create()
173 	pool->bitmap = devm_kzalloc(dev, bitmap_size, GFP_KERNEL);  in cpdma_desc_pool_create()
174 	if (!pool->bitmap)  in cpdma_desc_pool_create()
178 		pool->phys  = phys;  in cpdma_desc_pool_create()
179 		pool->iomap = ioremap(phys, size);  in cpdma_desc_pool_create()
180 		pool->hw_addr = hw_addr;  in cpdma_desc_pool_create()
182 		pool->cpumap = dma_alloc_coherent(dev, size, &pool->phys,  in cpdma_desc_pool_create()
184 		pool->iomap = pool->cpumap;  in cpdma_desc_pool_create()
185 		pool->hw_addr = pool->phys;  in cpdma_desc_pool_create()
188 	if (pool->iomap)  in cpdma_desc_pool_create()
189 		return pool;  in cpdma_desc_pool_create()
194 static void cpdma_desc_pool_destroy(struct cpdma_desc_pool *pool)  in cpdma_desc_pool_destroy()  argument
196 	if (!pool)  in cpdma_desc_pool_destroy()
199 	WARN_ON(pool->used_desc);  in cpdma_desc_pool_destroy()
200 	if (pool->cpumap) {  in cpdma_desc_pool_destroy()
201 		dma_free_coherent(pool->dev, pool->mem_size, pool->cpumap,  in cpdma_desc_pool_destroy()
202 				  pool->phys);  in cpdma_desc_pool_destroy()
204 		iounmap(pool->iomap);  in cpdma_desc_pool_destroy()
208 static inline dma_addr_t desc_phys(struct cpdma_desc_pool *pool,  in desc_phys()  argument
213 	return pool->hw_addr + (__force long)desc - (__force long)pool->iomap;  in desc_phys()
217 desc_from_phys(struct cpdma_desc_pool *pool, dma_addr_t dma)  in desc_from_phys()  argument
219 	return dma ? pool->iomap + dma - pool->hw_addr : NULL;  in desc_from_phys()
223 cpdma_desc_alloc(struct cpdma_desc_pool *pool, int num_desc, bool is_rx)  in cpdma_desc_alloc()  argument
231 	spin_lock_irqsave(&pool->lock, flags);  in cpdma_desc_alloc()
235 		desc_end = pool->num_desc/2;  in cpdma_desc_alloc()
237 		desc_start = pool->num_desc/2;  in cpdma_desc_alloc()
238 		desc_end = pool->num_desc;  in cpdma_desc_alloc()
241 	index = bitmap_find_next_zero_area(pool->bitmap,  in cpdma_desc_alloc()
244 		bitmap_set(pool->bitmap, index, num_desc);  in cpdma_desc_alloc()
245 		desc = pool->iomap + pool->desc_size * index;  in cpdma_desc_alloc()
246 		pool->used_desc++;  in cpdma_desc_alloc()
249 	spin_unlock_irqrestore(&pool->lock, flags);  in cpdma_desc_alloc()
253 static void cpdma_desc_free(struct cpdma_desc_pool *pool,  in cpdma_desc_free()  argument
258 	index = ((unsigned long)desc - (unsigned long)pool->iomap) /  in cpdma_desc_free()
259 		pool->desc_size;  in cpdma_desc_free()
260 	spin_lock_irqsave(&pool->lock, flags);  in cpdma_desc_free()
261 	bitmap_clear(pool->bitmap, index, num_desc);  in cpdma_desc_free()
262 	pool->used_desc--;  in cpdma_desc_free()
263 	spin_unlock_irqrestore(&pool->lock, flags);  in cpdma_desc_free()
279 	ctlr->pool = cpdma_desc_pool_create(ctlr->dev,  in cpdma_ctlr_create()
284 	if (!ctlr->pool)  in cpdma_ctlr_create()
458 	cpdma_desc_pool_destroy(ctlr->pool);  in cpdma_ctlr_destroy()
630 	struct cpdma_desc_pool		*pool = ctlr->pool;  in __cpdma_chan_submit()  local
634 	desc_dma = desc_phys(pool, desc);  in __cpdma_chan_submit()
678 	desc = cpdma_desc_alloc(ctlr->pool, 1, is_rx_chan(chan));  in cpdma_chan_submit()
693 		cpdma_desc_free(ctlr->pool, desc, 1);  in cpdma_chan_submit()
728 	struct cpdma_desc_pool	*pool = ctlr->pool;  in cpdma_check_free_tx_desc()  local
730 	spin_lock_irqsave(&pool->lock, flags);  in cpdma_check_free_tx_desc()
732 	index = bitmap_find_next_zero_area(pool->bitmap,  in cpdma_check_free_tx_desc()
733 				pool->num_desc, pool->num_desc/2, 1, 0);  in cpdma_check_free_tx_desc()
735 	if (index < pool->num_desc)  in cpdma_check_free_tx_desc()
740 	spin_unlock_irqrestore(&pool->lock, flags);  in cpdma_check_free_tx_desc()
750 	struct cpdma_desc_pool		*pool = ctlr->pool;  in __cpdma_chan_free()  local
760 	cpdma_desc_free(pool, desc, 1);  in __cpdma_chan_free()
770 	struct cpdma_desc_pool		*pool = ctlr->pool;  in __cpdma_chan_process()  local
782 	desc_dma = desc_phys(pool, desc);  in __cpdma_chan_process()
798 	chan->head = desc_from_phys(pool, desc_read(desc, hw_next));  in __cpdma_chan_process()
805 		chan_write(chan, hdp, desc_phys(pool, chan->head));  in __cpdma_chan_process()
842 	struct cpdma_desc_pool	*pool = ctlr->pool;  in cpdma_chan_start()  local
857 		chan_write(chan, hdp, desc_phys(pool, chan->head));  in cpdma_chan_start()
870 	struct cpdma_desc_pool	*pool = ctlr->pool;  in cpdma_chan_stop()  local
914 		chan->head = desc_from_phys(pool, next_dma);  in cpdma_chan_stop()