Lines Matching refs:atchan
85 static struct at_desc *atc_first_active(struct at_dma_chan *atchan) in atc_first_active() argument
87 return list_first_entry(&atchan->active_list, in atc_first_active()
91 static struct at_desc *atc_first_queued(struct at_dma_chan *atchan) in atc_first_queued() argument
93 return list_first_entry(&atchan->queue, in atc_first_queued()
132 static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) in atc_desc_get() argument
140 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
141 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_desc_get()
148 dev_dbg(chan2dev(&atchan->chan_common), in atc_desc_get()
151 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
152 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_get()
157 ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); in atc_desc_get()
159 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
160 atchan->descs_allocated++; in atc_desc_get()
161 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
163 dev_err(chan2dev(&atchan->chan_common), in atc_desc_get()
176 static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) in atc_desc_put() argument
182 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_put()
184 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
187 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_desc_put()
188 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
190 list_add(&desc->desc_node, &atchan->free_list); in atc_desc_put()
191 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_put()
225 static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) in atc_dostart() argument
227 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_dostart()
230 if (atc_chan_is_enabled(atchan)) { in atc_dostart()
231 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
233 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
235 channel_readl(atchan, SADDR), in atc_dostart()
236 channel_readl(atchan, DADDR), in atc_dostart()
237 channel_readl(atchan, CTRLA), in atc_dostart()
238 channel_readl(atchan, CTRLB), in atc_dostart()
239 channel_readl(atchan, DSCR)); in atc_dostart()
245 vdbg_dump_regs(atchan); in atc_dostart()
247 channel_writel(atchan, SADDR, 0); in atc_dostart()
248 channel_writel(atchan, DADDR, 0); in atc_dostart()
249 channel_writel(atchan, CTRLA, 0); in atc_dostart()
250 channel_writel(atchan, CTRLB, 0); in atc_dostart()
251 channel_writel(atchan, DSCR, first->txd.phys); in atc_dostart()
252 channel_writel(atchan, SPIP, ATC_SPIP_HOLE(first->src_hole) | in atc_dostart()
254 channel_writel(atchan, DPIP, ATC_DPIP_HOLE(first->dst_hole) | in atc_dostart()
256 dma_writel(atdma, CHER, atchan->mask); in atc_dostart()
258 vdbg_dump_regs(atchan); in atc_dostart()
266 static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan, in atc_get_desc_by_cookie() argument
271 list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) { in atc_get_desc_by_cookie()
276 list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { in atc_get_desc_by_cookie()
312 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_get_bytes_left() local
313 struct at_desc *desc_first = atc_first_active(atchan); in atc_get_bytes_left()
323 desc = atc_get_desc_by_cookie(atchan, cookie); in atc_get_bytes_left()
383 dscr = channel_readl(atchan, DSCR); in atc_get_bytes_left()
385 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
390 new_dscr = channel_readl(atchan, DSCR); in atc_get_bytes_left()
410 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
434 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
448 atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) in atc_chain_complete() argument
451 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_chain_complete()
453 dev_vdbg(chan2dev(&atchan->chan_common), in atc_chain_complete()
457 if (!atc_chan_is_cyclic(atchan)) in atc_chain_complete()
468 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_chain_complete()
470 list_move(&desc->desc_node, &atchan->free_list); in atc_chain_complete()
475 if (!atc_chan_is_cyclic(atchan)) { in atc_chain_complete()
499 static void atc_complete_all(struct at_dma_chan *atchan) in atc_complete_all() argument
504 dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); in atc_complete_all()
510 if (!list_empty(&atchan->queue)) in atc_complete_all()
511 atc_dostart(atchan, atc_first_queued(atchan)); in atc_complete_all()
513 list_splice_init(&atchan->active_list, &list); in atc_complete_all()
515 list_splice_init(&atchan->queue, &atchan->active_list); in atc_complete_all()
518 atc_chain_complete(atchan, desc); in atc_complete_all()
527 static void atc_advance_work(struct at_dma_chan *atchan) in atc_advance_work() argument
529 dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); in atc_advance_work()
531 if (atc_chan_is_enabled(atchan)) in atc_advance_work()
534 if (list_empty(&atchan->active_list) || in atc_advance_work()
535 list_is_singular(&atchan->active_list)) { in atc_advance_work()
536 atc_complete_all(atchan); in atc_advance_work()
538 atc_chain_complete(atchan, atc_first_active(atchan)); in atc_advance_work()
540 atc_dostart(atchan, atc_first_active(atchan)); in atc_advance_work()
551 static void atc_handle_error(struct at_dma_chan *atchan) in atc_handle_error() argument
561 bad_desc = atc_first_active(atchan); in atc_handle_error()
566 list_splice_init(&atchan->queue, atchan->active_list.prev); in atc_handle_error()
569 if (!list_empty(&atchan->active_list)) in atc_handle_error()
570 atc_dostart(atchan, atc_first_active(atchan)); in atc_handle_error()
579 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
581 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
583 atc_dump_lli(atchan, &bad_desc->lli); in atc_handle_error()
585 atc_dump_lli(atchan, &child->lli); in atc_handle_error()
588 atc_chain_complete(atchan, bad_desc); in atc_handle_error()
597 static void atc_handle_cyclic(struct at_dma_chan *atchan) in atc_handle_cyclic() argument
599 struct at_desc *first = atc_first_active(atchan); in atc_handle_cyclic()
604 dev_vdbg(chan2dev(&atchan->chan_common), in atc_handle_cyclic()
606 channel_readl(atchan, DSCR)); in atc_handle_cyclic()
616 struct at_dma_chan *atchan = (struct at_dma_chan *)data; in atc_tasklet() local
619 spin_lock_irqsave(&atchan->lock, flags); in atc_tasklet()
620 if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) in atc_tasklet()
621 atc_handle_error(atchan); in atc_tasklet()
622 else if (atc_chan_is_cyclic(atchan)) in atc_tasklet()
623 atc_handle_cyclic(atchan); in atc_tasklet()
625 atc_advance_work(atchan); in atc_tasklet()
627 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tasklet()
633 struct at_dma_chan *atchan; in at_dma_interrupt() local
651 atchan = &atdma->chan[i]; in at_dma_interrupt()
656 AT_DMA_RES(i) | atchan->mask); in at_dma_interrupt()
658 set_bit(ATC_IS_ERROR, &atchan->status); in at_dma_interrupt()
660 tasklet_schedule(&atchan->tasklet); in at_dma_interrupt()
684 struct at_dma_chan *atchan = to_at_dma_chan(tx->chan); in atc_tx_submit() local
688 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_submit()
691 if (list_empty(&atchan->active_list)) { in atc_tx_submit()
694 atc_dostart(atchan, desc); in atc_tx_submit()
695 list_add_tail(&desc->desc_node, &atchan->active_list); in atc_tx_submit()
699 list_add_tail(&desc->desc_node, &atchan->queue); in atc_tx_submit()
702 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_submit()
718 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_interleaved() local
777 desc = atc_desc_get(atchan); in atc_prep_dma_interleaved()
816 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_memcpy() local
853 desc = atc_desc_get(atchan); in atc_prep_dma_memcpy()
880 atc_desc_put(atchan, first); in atc_prep_dma_memcpy()
889 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_create_memset_desc() local
906 desc = atc_desc_get(atchan); in atc_create_memset_desc()
995 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_memset_sg() local
1062 atc_desc_put(atchan, first); in atc_prep_dma_memset_sg()
1080 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_slave_sg() local
1082 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_slave_sg()
1115 | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if); in atc_prep_slave_sg()
1122 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
1155 | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if); in atc_prep_slave_sg()
1163 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
1209 atc_desc_put(atchan, first); in atc_prep_slave_sg()
1228 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_sg() local
1306 desc = atc_desc_get(atchan); in atc_prep_dma_sg()
1341 atc_desc_put(atchan, first); in atc_prep_dma_sg()
1375 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_dma_cyclic_fill_desc() local
1376 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_dma_cyclic_fill_desc()
1394 | ATC_SIF(atchan->mem_if) in atc_dma_cyclic_fill_desc()
1395 | ATC_DIF(atchan->per_if); in atc_dma_cyclic_fill_desc()
1406 | ATC_SIF(atchan->per_if) in atc_dma_cyclic_fill_desc()
1407 | ATC_DIF(atchan->mem_if); in atc_dma_cyclic_fill_desc()
1432 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_cyclic() local
1434 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_dma_cyclic()
1452 was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
1474 desc = atc_desc_get(atchan); in atc_prep_dma_cyclic()
1496 atc_desc_put(atchan, first); in atc_prep_dma_cyclic()
1498 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
1505 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_config() local
1513 memcpy(&atchan->dma_sconfig, sconfig, sizeof(*sconfig)); in atc_config()
1515 convert_burst(&atchan->dma_sconfig.src_maxburst); in atc_config()
1516 convert_burst(&atchan->dma_sconfig.dst_maxburst); in atc_config()
1523 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_pause() local
1525 int chan_id = atchan->chan_common.chan_id; in atc_pause()
1532 spin_lock_irqsave(&atchan->lock, flags); in atc_pause()
1535 set_bit(ATC_IS_PAUSED, &atchan->status); in atc_pause()
1537 spin_unlock_irqrestore(&atchan->lock, flags); in atc_pause()
1544 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_resume() local
1546 int chan_id = atchan->chan_common.chan_id; in atc_resume()
1553 if (!atc_chan_is_paused(atchan)) in atc_resume()
1556 spin_lock_irqsave(&atchan->lock, flags); in atc_resume()
1559 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_resume()
1561 spin_unlock_irqrestore(&atchan->lock, flags); in atc_resume()
1568 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_terminate_all() local
1570 int chan_id = atchan->chan_common.chan_id; in atc_terminate_all()
1584 spin_lock_irqsave(&atchan->lock, flags); in atc_terminate_all()
1587 dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); in atc_terminate_all()
1590 while (dma_readl(atdma, CHSR) & atchan->mask) in atc_terminate_all()
1594 list_splice_init(&atchan->queue, &list); in atc_terminate_all()
1595 list_splice_init(&atchan->active_list, &list); in atc_terminate_all()
1599 atc_chain_complete(atchan, desc); in atc_terminate_all()
1601 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_terminate_all()
1603 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_terminate_all()
1605 spin_unlock_irqrestore(&atchan->lock, flags); in atc_terminate_all()
1625 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_tx_status() local
1640 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_status()
1645 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_status()
1666 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_issue_pending() local
1672 if (atc_chan_is_cyclic(atchan)) in atc_issue_pending()
1675 spin_lock_irqsave(&atchan->lock, flags); in atc_issue_pending()
1676 atc_advance_work(atchan); in atc_issue_pending()
1677 spin_unlock_irqrestore(&atchan->lock, flags); in atc_issue_pending()
1689 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_alloc_chan_resources() local
1701 if (atc_chan_is_enabled(atchan)) { in atc_alloc_chan_resources()
1723 if (!list_empty(&atchan->free_list)) in atc_alloc_chan_resources()
1724 return atchan->descs_allocated; in atc_alloc_chan_resources()
1737 spin_lock_irqsave(&atchan->lock, flags); in atc_alloc_chan_resources()
1738 atchan->descs_allocated = i; in atc_alloc_chan_resources()
1739 list_splice(&tmp_list, &atchan->free_list); in atc_alloc_chan_resources()
1741 spin_unlock_irqrestore(&atchan->lock, flags); in atc_alloc_chan_resources()
1744 channel_writel(atchan, CFG, cfg); in atc_alloc_chan_resources()
1748 atchan->descs_allocated); in atc_alloc_chan_resources()
1750 return atchan->descs_allocated; in atc_alloc_chan_resources()
1759 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_free_chan_resources() local
1765 atchan->descs_allocated); in atc_free_chan_resources()
1768 BUG_ON(!list_empty(&atchan->active_list)); in atc_free_chan_resources()
1769 BUG_ON(!list_empty(&atchan->queue)); in atc_free_chan_resources()
1770 BUG_ON(atc_chan_is_enabled(atchan)); in atc_free_chan_resources()
1772 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_free_chan_resources()
1778 list_splice_init(&atchan->free_list, &list); in atc_free_chan_resources()
1779 atchan->descs_allocated = 0; in atc_free_chan_resources()
1780 atchan->status = 0; in atc_free_chan_resources()
1802 struct at_dma_chan *atchan; in at_dma_xlate() local
1850 atchan = to_at_dma_chan(chan); in at_dma_xlate()
1851 atchan->per_if = dma_spec->args[0] & 0xff; in at_dma_xlate()
1852 atchan->mem_if = (dma_spec->args[0] >> 16) & 0xff; in at_dma_xlate()
2032 struct at_dma_chan *atchan = &atdma->chan[i]; in at_dma_probe() local
2034 atchan->mem_if = AT_DMA_MEM_IF; in at_dma_probe()
2035 atchan->per_if = AT_DMA_PER_IF; in at_dma_probe()
2036 atchan->chan_common.device = &atdma->dma_common; in at_dma_probe()
2037 dma_cookie_init(&atchan->chan_common); in at_dma_probe()
2038 list_add_tail(&atchan->chan_common.device_node, in at_dma_probe()
2041 atchan->ch_regs = atdma->regs + ch_regs(i); in at_dma_probe()
2042 spin_lock_init(&atchan->lock); in at_dma_probe()
2043 atchan->mask = 1 << i; in at_dma_probe()
2045 INIT_LIST_HEAD(&atchan->active_list); in at_dma_probe()
2046 INIT_LIST_HEAD(&atchan->queue); in at_dma_probe()
2047 INIT_LIST_HEAD(&atchan->free_list); in at_dma_probe()
2049 tasklet_init(&atchan->tasklet, atc_tasklet, in at_dma_probe()
2050 (unsigned long)atchan); in at_dma_probe()
2155 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_remove() local
2160 tasklet_kill(&atchan->tasklet); in at_dma_remove()
2194 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_prepare() local
2196 if (atc_chan_is_enabled(atchan) && !atc_chan_is_cyclic(atchan)) in at_dma_prepare()
2202 static void atc_suspend_cyclic(struct at_dma_chan *atchan) in atc_suspend_cyclic() argument
2204 struct dma_chan *chan = &atchan->chan_common; in atc_suspend_cyclic()
2208 if (!atc_chan_is_paused(atchan)) { in atc_suspend_cyclic()
2216 atchan->save_dscr = channel_readl(atchan, DSCR); in atc_suspend_cyclic()
2218 vdbg_dump_regs(atchan); in atc_suspend_cyclic()
2230 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_suspend_noirq() local
2232 if (atc_chan_is_cyclic(atchan)) in at_dma_suspend_noirq()
2233 atc_suspend_cyclic(atchan); in at_dma_suspend_noirq()
2234 atchan->save_cfg = channel_readl(atchan, CFG); in at_dma_suspend_noirq()
2244 static void atc_resume_cyclic(struct at_dma_chan *atchan) in atc_resume_cyclic() argument
2246 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_resume_cyclic()
2250 channel_writel(atchan, SADDR, 0); in atc_resume_cyclic()
2251 channel_writel(atchan, DADDR, 0); in atc_resume_cyclic()
2252 channel_writel(atchan, CTRLA, 0); in atc_resume_cyclic()
2253 channel_writel(atchan, CTRLB, 0); in atc_resume_cyclic()
2254 channel_writel(atchan, DSCR, atchan->save_dscr); in atc_resume_cyclic()
2255 dma_writel(atdma, CHER, atchan->mask); in atc_resume_cyclic()
2260 vdbg_dump_regs(atchan); in atc_resume_cyclic()
2281 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_resume_noirq() local
2283 channel_writel(atchan, CFG, atchan->save_cfg); in at_dma_resume_noirq()
2284 if (atc_chan_is_cyclic(atchan)) in at_dma_resume_noirq()
2285 atc_resume_cyclic(atchan); in at_dma_resume_noirq()