267 #define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg))  argument
268 #define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg))  argument
290 static inline int at_xdmac_chan_is_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_cyclic()  argument
292 	return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_chan_is_cyclic()
295 static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_paused()  argument
297 	return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_chan_is_paused()
322 static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_enabled()  argument
324 	return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask;  in at_xdmac_chan_is_enabled()
339 static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,  in at_xdmac_start_xfer()  argument
342 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()
345 	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);  in at_xdmac_start_xfer()
347 	if (at_xdmac_chan_is_enabled(atchan))  in at_xdmac_start_xfer()
355 	      | AT_XDMAC_CNDA_NDAIF(atchan->memif);  in at_xdmac_start_xfer()
356 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);  in at_xdmac_start_xfer()
363 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
375 	at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg);  in at_xdmac_start_xfer()
380 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg);  in at_xdmac_start_xfer()
382 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
384 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
385 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
386 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
387 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
388 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
389 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
391 	at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff);  in at_xdmac_start_xfer()
397 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
398 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
401 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
403 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);  in at_xdmac_start_xfer()
404 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
405 		 "%s: enable channel (0x%08x)\n", __func__, atchan->mask);  in at_xdmac_start_xfer()
407 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in at_xdmac_start_xfer()
409 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
411 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
412 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
413 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
414 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
415 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
416 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
423 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(tx->chan);  in at_xdmac_tx_submit()  local
427 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
431 		 __func__, atchan, desc);  in at_xdmac_tx_submit()
432 	list_add_tail(&desc->xfer_node, &atchan->xfers_list);  in at_xdmac_tx_submit()
433 	if (list_is_singular(&atchan->xfers_list))  in at_xdmac_tx_submit()
434 		at_xdmac_start_xfer(atchan, desc);  in at_xdmac_tx_submit()
436 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
469 static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan)  in at_xdmac_get_desc()  argument
473 	if (list_empty(&atchan->free_descs_list)) {  in at_xdmac_get_desc()
474 		desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT);  in at_xdmac_get_desc()
476 		desc = list_first_entry(&atchan->free_descs_list,  in at_xdmac_get_desc()
516 	struct at_xdmac_chan	*atchan;  in at_xdmac_xlate()  local
531 	atchan = to_at_xdmac_chan(chan);  in at_xdmac_xlate()
532 	atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]);  in at_xdmac_xlate()
533 	atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]);  in at_xdmac_xlate()
534 	atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]);  in at_xdmac_xlate()
536 		 atchan->memif, atchan->perif, atchan->perid);  in at_xdmac_xlate()
544 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_compute_chan_conf()  local
548 		atchan->cfg =  in at_xdmac_compute_chan_conf()
549 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
552 			| AT_XDMAC_CC_DIF(atchan->memif)  in at_xdmac_compute_chan_conf()
553 			| AT_XDMAC_CC_SIF(atchan->perif)  in at_xdmac_compute_chan_conf()
558 		csize = ffs(atchan->sconfig.src_maxburst) - 1;  in at_xdmac_compute_chan_conf()
563 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
564 		dwidth = ffs(atchan->sconfig.src_addr_width) - 1;  in at_xdmac_compute_chan_conf()
569 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
571 		atchan->cfg =  in at_xdmac_compute_chan_conf()
572 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
575 			| AT_XDMAC_CC_DIF(atchan->perif)  in at_xdmac_compute_chan_conf()
576 			| AT_XDMAC_CC_SIF(atchan->memif)  in at_xdmac_compute_chan_conf()
581 		csize = ffs(atchan->sconfig.dst_maxburst) - 1;  in at_xdmac_compute_chan_conf()
586 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
587 		dwidth = ffs(atchan->sconfig.dst_addr_width) - 1;  in at_xdmac_compute_chan_conf()
592 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
595 	dev_dbg(chan2dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);  in at_xdmac_compute_chan_conf()
621 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_set_slave_config()  local
628 	memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig));  in at_xdmac_set_slave_config()
638 	struct at_xdmac_chan		*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_slave_sg()  local
660 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
679 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_slave_sg()
683 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_slave_sg()
689 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_slave_sg()
693 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_slave_sg()
695 		dwidth = at_xdmac_get_dwidth(atchan->cfg);  in at_xdmac_prep_slave_sg()
703 		desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |  in at_xdmac_prep_slave_sg()
730 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
740 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_cyclic()  local
755 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {  in at_xdmac_prep_dma_cyclic()
766 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
767 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_cyclic()
771 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_dma_cyclic()
772 			spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
775 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
781 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_dma_cyclic()
785 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_dma_cyclic()
787 		desc->lld.mbr_cfg = atchan->cfg;  in at_xdmac_prep_dma_cyclic()
849 				struct at_xdmac_chan *atchan,  in at_xdmac_interleaved_queue_desc()  argument
900 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
901 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_interleaved_queue_desc()
902 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
940 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_interleaved()  local
965 		first = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
994 			desc = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
1000 						 &atchan->free_descs_list);  in at_xdmac_prep_interleaved()
1033 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memcpy()  local
1068 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1069 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_memcpy()
1070 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1074 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_dma_memcpy()
1130 							 struct at_xdmac_chan *atchan,  in at_xdmac_memset_create_desc()  argument
1165 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1166 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_memset_create_desc()
1167 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1197 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset()  local
1206 	desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value);  in at_xdmac_prep_dma_memset()
1221 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset_sg()  local
1239 		desc = at_xdmac_memset_create_desc(chan, atchan,  in at_xdmac_prep_dma_memset_sg()
1245 					 &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1294 					      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1357 				      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1382 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_tx_status()  local
1383 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()
1399 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_tx_status()
1401 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);  in at_xdmac_tx_status()
1420 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1421 		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))  in at_xdmac_tx_status()
1441 	cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1443 	cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC);  in at_xdmac_tx_status()
1446 		check_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1453 		cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC);  in at_xdmac_tx_status()
1482 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_tx_status()
1487 static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan,  in at_xdmac_remove_xfer()  argument
1490 	dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_remove_xfer()
1497 	list_splice_init(&desc->descs_list, &atchan->free_descs_list);  in at_xdmac_remove_xfer()
1500 static void at_xdmac_advance_work(struct at_xdmac_chan *atchan)  in at_xdmac_advance_work()  argument
1505 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_advance_work()
1511 	if (!at_xdmac_chan_is_enabled(atchan) && !list_empty(&atchan->xfers_list)) {  in at_xdmac_advance_work()
1512 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_advance_work()
1515 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_advance_work()
1517 			at_xdmac_start_xfer(atchan, desc);  in at_xdmac_advance_work()
1520 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_advance_work()
1523 static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_handle_cyclic()  argument
1528 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);  in at_xdmac_handle_cyclic()
1537 	struct at_xdmac_chan	*atchan = (struct at_xdmac_chan *)data;  in at_xdmac_tasklet()  local
1541 	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08lx\n",  in at_xdmac_tasklet()
1542 		 __func__, atchan->status);  in at_xdmac_tasklet()
1548 	if (at_xdmac_chan_is_cyclic(atchan)) {  in at_xdmac_tasklet()
1549 		at_xdmac_handle_cyclic(atchan);  in at_xdmac_tasklet()
1550 	} else if ((atchan->status & AT_XDMAC_CIS_LIS)  in at_xdmac_tasklet()
1551 		   || (atchan->status & error_mask)) {  in at_xdmac_tasklet()
1554 		if (atchan->status & AT_XDMAC_CIS_RBEIS)  in at_xdmac_tasklet()
1555 			dev_err(chan2dev(&atchan->chan), "read bus error!!!");  in at_xdmac_tasklet()
1556 		if (atchan->status & AT_XDMAC_CIS_WBEIS)  in at_xdmac_tasklet()
1557 			dev_err(chan2dev(&atchan->chan), "write bus error!!!");  in at_xdmac_tasklet()
1558 		if (atchan->status & AT_XDMAC_CIS_ROIS)  in at_xdmac_tasklet()
1559 			dev_err(chan2dev(&atchan->chan), "request overflow error!!!");  in at_xdmac_tasklet()
1561 		spin_lock_bh(&atchan->lock);  in at_xdmac_tasklet()
1562 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_tasklet()
1565 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_tasklet()
1570 		at_xdmac_remove_xfer(atchan, desc);  in at_xdmac_tasklet()
1571 		spin_unlock_bh(&atchan->lock);  in at_xdmac_tasklet()
1573 		if (!at_xdmac_chan_is_cyclic(atchan)) {  in at_xdmac_tasklet()
1581 		at_xdmac_advance_work(atchan);  in at_xdmac_tasklet()
1588 	struct at_xdmac_chan	*atchan;  in at_xdmac_interrupt()  local
1610 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1611 			chan_imr = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in at_xdmac_interrupt()
1612 			chan_status = at_xdmac_chan_read(atchan, AT_XDMAC_CIS);  in at_xdmac_interrupt()
1613 			atchan->status = chan_status & chan_imr;  in at_xdmac_interrupt()
1617 			dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_interrupt()
1620 				 at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_interrupt()
1621 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_interrupt()
1622 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_interrupt()
1623 				 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_interrupt()
1624 				 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_interrupt()
1625 				 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_interrupt()
1627 			if (atchan->status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS))  in at_xdmac_interrupt()
1628 				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_interrupt()
1630 			tasklet_schedule(&atchan->tasklet);  in at_xdmac_interrupt()
1641 	struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);  in at_xdmac_issue_pending()  local
1643 	dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);  in at_xdmac_issue_pending()
1645 	if (!at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_issue_pending()
1646 		at_xdmac_advance_work(atchan);  in at_xdmac_issue_pending()
1654 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_config()  local
1660 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_config()
1662 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_config()
1669 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_pause()  local
1670 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()
1675 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status))  in at_xdmac_device_pause()
1678 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_pause()
1679 	at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);  in at_xdmac_device_pause()
1680 	while (at_xdmac_chan_read(atchan, AT_XDMAC_CC)  in at_xdmac_device_pause()
1683 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_pause()
1690 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_resume()  local
1691 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()
1696 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_resume()
1697 	if (!at_xdmac_chan_is_paused(atchan)) {  in at_xdmac_device_resume()
1698 		spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume()
1702 	at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);  in at_xdmac_device_resume()
1703 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_resume()
1704 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume()
1712 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_terminate_all()  local
1713 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()
1718 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
1719 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_device_terminate_all()
1720 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_device_terminate_all()
1724 	list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node)  in at_xdmac_device_terminate_all()
1725 		at_xdmac_remove_xfer(atchan, desc);  in at_xdmac_device_terminate_all()
1727 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_terminate_all()
1728 	clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_device_terminate_all()
1729 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
1736 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_alloc_chan_resources()  local
1741 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_alloc_chan_resources()
1743 	if (at_xdmac_chan_is_enabled(atchan)) {  in at_xdmac_alloc_chan_resources()
1750 	if (!list_empty(&atchan->free_descs_list)) {  in at_xdmac_alloc_chan_resources()
1764 		list_add_tail(&desc->desc_node, &atchan->free_descs_list);  in at_xdmac_alloc_chan_resources()
1772 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_alloc_chan_resources()
1778 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_free_chan_resources()  local
1782 	list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) {  in at_xdmac_free_chan_resources()
1799 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_prepare()  local
1802 		if (at_xdmac_chan_is_enabled(atchan) && !at_xdmac_chan_is_cyclic(atchan))  in atmel_xdmac_prepare()
1819 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_suspend()  local
1821 		atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC);  in atmel_xdmac_suspend()
1822 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_suspend()
1823 			if (!at_xdmac_chan_is_paused(atchan))  in atmel_xdmac_suspend()
1825 			atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in atmel_xdmac_suspend()
1826 			atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA);  in atmel_xdmac_suspend()
1827 			atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC);  in atmel_xdmac_suspend()
1841 	struct at_xdmac_chan	*atchan;  in atmel_xdmac_resume()  local
1849 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1850 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in atmel_xdmac_resume()
1857 		atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_resume()
1858 		at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc);  in atmel_xdmac_resume()
1859 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_resume()
1860 			if (at_xdmac_chan_is_paused(atchan))  in atmel_xdmac_resume()
1862 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda);  in atmel_xdmac_resume()
1863 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc);  in atmel_xdmac_resume()
1864 			at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim);  in atmel_xdmac_resume()
1866 			at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in atmel_xdmac_resume()
1982 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()  local
1984 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
1985 		list_add_tail(&atchan->chan.device_node,  in at_xdmac_probe()
1988 		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);  in at_xdmac_probe()
1989 		atchan->mask = 1 << i;  in at_xdmac_probe()
1991 		spin_lock_init(&atchan->lock);  in at_xdmac_probe()
1992 		INIT_LIST_HEAD(&atchan->xfers_list);  in at_xdmac_probe()
1993 		INIT_LIST_HEAD(&atchan->free_descs_list);  in at_xdmac_probe()
1994 		tasklet_init(&atchan->tasklet, at_xdmac_tasklet,  in at_xdmac_probe()
1995 			     (unsigned long)atchan);  in at_xdmac_probe()
1998 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in at_xdmac_probe()
2045 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()  local
2047 		tasklet_kill(&atchan->tasklet);  in at_xdmac_remove()
2048 		at_xdmac_free_chan_resources(&atchan->chan);  in at_xdmac_remove()