1451 	struct dma_pl330_chan *pch;  in dma_pl330_rqcb()  local
1457 	pch = desc->pchan;  in dma_pl330_rqcb()
1460 	if (!pch)  in dma_pl330_rqcb()
1463 	spin_lock_irqsave(&pch->lock, flags);  in dma_pl330_rqcb()
1467 	spin_unlock_irqrestore(&pch->lock, flags);  in dma_pl330_rqcb()
1469 	tasklet_schedule(&pch->task);  in dma_pl330_rqcb()
1945 static inline void fill_queue(struct dma_pl330_chan *pch)  in fill_queue()  argument
1950 	list_for_each_entry(desc, &pch->work_list, node) {  in fill_queue()
1956 		ret = pl330_submit_req(pch->thread, desc);  in fill_queue()
1965 			dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n",  in fill_queue()
1967 			tasklet_schedule(&pch->task);  in fill_queue()
1974 	struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;  in pl330_tasklet()  local
1979 	spin_lock_irqsave(&pch->lock, flags);  in pl330_tasklet()
1982 	list_for_each_entry_safe(desc, _dt, &pch->work_list, node)  in pl330_tasklet()
1984 			if (!pch->cyclic)  in pl330_tasklet()
1986 			list_move_tail(&desc->node, &pch->completed_list);  in pl330_tasklet()
1990 	fill_queue(pch);  in pl330_tasklet()
1992 	if (list_empty(&pch->work_list)) {  in pl330_tasklet()
1993 		spin_lock(&pch->thread->dmac->lock);  in pl330_tasklet()
1994 		_stop(pch->thread);  in pl330_tasklet()
1995 		spin_unlock(&pch->thread->dmac->lock);  in pl330_tasklet()
1999 		spin_lock(&pch->thread->dmac->lock);  in pl330_tasklet()
2000 		_start(pch->thread);  in pl330_tasklet()
2001 		spin_unlock(&pch->thread->dmac->lock);  in pl330_tasklet()
2004 	while (!list_empty(&pch->completed_list)) {  in pl330_tasklet()
2008 		desc = list_first_entry(&pch->completed_list,  in pl330_tasklet()
2014 		if (pch->cyclic) {  in pl330_tasklet()
2016 			list_move_tail(&desc->node, &pch->work_list);  in pl330_tasklet()
2018 				spin_lock(&pch->thread->dmac->lock);  in pl330_tasklet()
2019 				_start(pch->thread);  in pl330_tasklet()
2020 				spin_unlock(&pch->thread->dmac->lock);  in pl330_tasklet()
2025 			list_move_tail(&desc->node, &pch->dmac->desc_pool);  in pl330_tasklet()
2031 			spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tasklet()
2033 			spin_lock_irqsave(&pch->lock, flags);  in pl330_tasklet()
2036 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tasklet()
2040 		pm_runtime_mark_last_busy(pch->dmac->ddma.dev);  in pl330_tasklet()
2041 		pm_runtime_put_autosuspend(pch->dmac->ddma.dev);  in pl330_tasklet()
2079 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_alloc_chan_resources()  local
2080 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_alloc_chan_resources()
2083 	spin_lock_irqsave(&pch->lock, flags);  in pl330_alloc_chan_resources()
2086 	pch->cyclic = false;  in pl330_alloc_chan_resources()
2088 	pch->thread = pl330_request_channel(pl330);  in pl330_alloc_chan_resources()
2089 	if (!pch->thread) {  in pl330_alloc_chan_resources()
2090 		spin_unlock_irqrestore(&pch->lock, flags);  in pl330_alloc_chan_resources()
2094 	tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);  in pl330_alloc_chan_resources()
2096 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_alloc_chan_resources()
2104 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_config()  local
2108 			pch->fifo_addr = slave_config->dst_addr;  in pl330_config()
2110 			pch->burst_sz = __ffs(slave_config->dst_addr_width);  in pl330_config()
2112 			pch->burst_len = slave_config->dst_maxburst;  in pl330_config()
2115 			pch->fifo_addr = slave_config->src_addr;  in pl330_config()
2117 			pch->burst_sz = __ffs(slave_config->src_addr_width);  in pl330_config()
2119 			pch->burst_len = slave_config->src_maxburst;  in pl330_config()
2127 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_terminate_all()  local
2130 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_terminate_all()
2134 	spin_lock_irqsave(&pch->lock, flags);  in pl330_terminate_all()
2136 	_stop(pch->thread);  in pl330_terminate_all()
2139 	pch->thread->req[0].desc = NULL;  in pl330_terminate_all()
2140 	pch->thread->req[1].desc = NULL;  in pl330_terminate_all()
2141 	pch->thread->req_running = -1;  in pl330_terminate_all()
2144 	list_for_each_entry(desc, &pch->submitted_list, node) {  in pl330_terminate_all()
2149 	list_for_each_entry(desc, &pch->work_list , node) {  in pl330_terminate_all()
2154 	list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool);  in pl330_terminate_all()
2155 	list_splice_tail_init(&pch->work_list, &pl330->desc_pool);  in pl330_terminate_all()
2156 	list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);  in pl330_terminate_all()
2157 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_terminate_all()
2173 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_pause()  local
2174 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_pause()
2178 	spin_lock_irqsave(&pch->lock, flags);  in pl330_pause()
2181 	_stop(pch->thread);  in pl330_pause()
2184 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_pause()
2193 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_free_chan_resources()  local
2196 	tasklet_kill(&pch->task);  in pl330_free_chan_resources()
2198 	pm_runtime_get_sync(pch->dmac->ddma.dev);  in pl330_free_chan_resources()
2199 	spin_lock_irqsave(&pch->lock, flags);  in pl330_free_chan_resources()
2201 	pl330_release_channel(pch->thread);  in pl330_free_chan_resources()
2202 	pch->thread = NULL;  in pl330_free_chan_resources()
2204 	if (pch->cyclic)  in pl330_free_chan_resources()
2205 		list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);  in pl330_free_chan_resources()
2207 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_free_chan_resources()
2208 	pm_runtime_mark_last_busy(pch->dmac->ddma.dev);  in pl330_free_chan_resources()
2209 	pm_runtime_put_autosuspend(pch->dmac->ddma.dev);  in pl330_free_chan_resources()
2212 static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch,  in pl330_get_current_xferred_count()  argument
2215 	struct pl330_thread *thrd = pch->thread;  in pl330_get_current_xferred_count()
2216 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_get_current_xferred_count()
2229 	pm_runtime_mark_last_busy(pch->dmac->ddma.dev);  in pl330_get_current_xferred_count()
2241 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_tx_status()  local
2252 	spin_lock_irqsave(&pch->lock, flags);  in pl330_tx_status()
2254 	if (pch->thread->req_running != -1)  in pl330_tx_status()
2255 		running = pch->thread->req[pch->thread->req_running].desc;  in pl330_tx_status()
2258 	list_for_each_entry(desc, &pch->work_list, node) {  in pl330_tx_status()
2263 				pl330_get_current_xferred_count(pch, desc);  in pl330_tx_status()
2284 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tx_status()
2294 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_issue_pending()  local
2297 	spin_lock_irqsave(&pch->lock, flags);  in pl330_issue_pending()
2298 	if (list_empty(&pch->work_list)) {  in pl330_issue_pending()
2304 		WARN_ON(list_empty(&pch->submitted_list));  in pl330_issue_pending()
2305 		pm_runtime_get_sync(pch->dmac->ddma.dev);  in pl330_issue_pending()
2307 	list_splice_tail_init(&pch->submitted_list, &pch->work_list);  in pl330_issue_pending()
2308 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_issue_pending()
2310 	pl330_tasklet((unsigned long)pch);  in pl330_issue_pending()
2321 	struct dma_pl330_chan *pch = to_pchan(tx->chan);  in pl330_tx_submit()  local
2325 	spin_lock_irqsave(&pch->lock, flags);  in pl330_tx_submit()
2330 		if (pch->cyclic) {  in pl330_tx_submit()
2338 		list_move_tail(&desc->node, &pch->submitted_list);  in pl330_tx_submit()
2343 	list_add_tail(&last->node, &pch->submitted_list);  in pl330_tx_submit()
2344 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tx_submit()
2404 static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)  in pl330_get_desc()  argument
2406 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_get_desc()
2407 	u8 *peri_id = pch->chan.private;  in pl330_get_desc()
2421 			dev_err(pch->dmac->ddma.dev,  in pl330_get_desc()
2428 	desc->pchan = pch;  in pl330_get_desc()
2432 	desc->peri = peri_id ? pch->chan.chan_id : 0;  in pl330_get_desc()
2433 	desc->rqcfg.pcfg = &pch->dmac->pcfg;  in pl330_get_desc()
2435 	dma_async_tx_descriptor_init(&desc->txd, &pch->chan);  in pl330_get_desc()
2449 __pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,  in __pl330_prep_dma_memcpy()  argument
2452 	struct dma_pl330_desc *desc = pl330_get_desc(pch);  in __pl330_prep_dma_memcpy()
2455 		dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",  in __pl330_prep_dma_memcpy()
2478 	struct dma_pl330_chan *pch = desc->pchan;  in get_burst_len()  local
2479 	struct pl330_dmac *pl330 = pch->dmac;  in get_burst_len()
2505 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_prep_dma_cyclic()  local
2506 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_prep_dma_cyclic()
2515 		dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n",  in pl330_prep_dma_cyclic()
2521 		desc = pl330_get_desc(pch);  in pl330_prep_dma_cyclic()
2523 			dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",  in pl330_prep_dma_cyclic()
2549 			dst = pch->fifo_addr;  in pl330_prep_dma_cyclic()
2554 			src = pch->fifo_addr;  in pl330_prep_dma_cyclic()
2562 		desc->rqcfg.brst_size = pch->burst_sz;  in pl330_prep_dma_cyclic()
2578 	pch->cyclic = true;  in pl330_prep_dma_cyclic()
2589 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_prep_dma_memcpy()  local
2593 	if (unlikely(!pch || !len))  in pl330_prep_dma_memcpy()
2596 	pl330 = pch->dmac;  in pl330_prep_dma_memcpy()
2598 	desc = __pl330_prep_dma_memcpy(pch, dst, src, len);  in pl330_prep_dma_memcpy()
2664 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_prep_slave_sg()  local
2669 	if (unlikely(!pch || !sgl || !sg_len))  in pl330_prep_slave_sg()
2672 	addr = pch->fifo_addr;  in pl330_prep_slave_sg()
2678 		desc = pl330_get_desc(pch);  in pl330_prep_slave_sg()
2680 			struct pl330_dmac *pl330 = pch->dmac;  in pl330_prep_slave_sg()
2682 			dev_err(pch->dmac->ddma.dev,  in pl330_prep_slave_sg()
2707 		desc->rqcfg.brst_size = pch->burst_sz;  in pl330_prep_slave_sg()
2779 	struct dma_pl330_chan *pch, *_p;  in pl330_probe()  local
2847 	pl330->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);  in pl330_probe()
2855 		pch = &pl330->peripherals[i];  in pl330_probe()
2857 			pch->chan.private = pdat ? &pdat->peri_id[i] : NULL;  in pl330_probe()
2859 			pch->chan.private = adev->dev.of_node;  in pl330_probe()
2861 		INIT_LIST_HEAD(&pch->submitted_list);  in pl330_probe()
2862 		INIT_LIST_HEAD(&pch->work_list);  in pl330_probe()
2863 		INIT_LIST_HEAD(&pch->completed_list);  in pl330_probe()
2864 		spin_lock_init(&pch->lock);  in pl330_probe()
2865 		pch->thread = NULL;  in pl330_probe()
2866 		pch->chan.device = pd;  in pl330_probe()
2867 		pch->dmac = pl330;  in pl330_probe()
2870 		list_add_tail(&pch->chan.device_node, &pd->channels);  in pl330_probe()
2941 	list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,  in pl330_probe()
2945 		list_del(&pch->chan.device_node);  in pl330_probe()
2948 		if (pch->thread) {  in pl330_probe()
2949 			pl330_terminate_all(&pch->chan);  in pl330_probe()
2950 			pl330_free_chan_resources(&pch->chan);  in pl330_probe()
2962 	struct dma_pl330_chan *pch, *_p;  in pl330_remove()  local
2972 	list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,  in pl330_remove()
2976 		list_del(&pch->chan.device_node);  in pl330_remove()
2979 		if (pch->thread) {  in pl330_remove()
2980 			pl330_terminate_all(&pch->chan);  in pl330_remove()
2981 			pl330_free_chan_resources(&pch->chan);  in pl330_remove()