132 	struct dw_dma *dw = to_dw_dma(dwc->chan.device);  in dwc_initialize()  local
146 	channel_set_bit(dw, MASK.XFER, dwc->mask);  in dwc_initialize()
147 	channel_set_bit(dw, MASK.ERROR, dwc->mask);  in dwc_initialize()
180 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)  in dwc_chan_disable()  argument
182 	channel_clear_bit(dw, CH_EN, dwc->mask);  in dwc_chan_disable()
183 	while (dma_readl(dw, CH_EN) & dwc->mask)  in dwc_chan_disable()
193 	struct dw_dma	*dw = to_dw_dma(dwc->chan.device);  in dwc_do_single_block()  local
206 	channel_set_bit(dw, CH_EN, dwc->mask);  in dwc_do_single_block()
215 	struct dw_dma	*dw = to_dw_dma(dwc->chan.device);  in dwc_dostart()  local
219 	if (dma_readl(dw, CH_EN) & dwc->mask) {  in dwc_dostart()
255 	channel_set_bit(dw, CH_EN, dwc->mask);  in dwc_dostart()
307 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)  in dwc_complete_all()  argument
314 	if (dma_readl(dw, CH_EN) & dwc->mask) {  in dwc_complete_all()
319 		dwc_chan_disable(dw, dwc);  in dwc_complete_all()
344 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)  in dwc_scan_descriptors()  argument
354 	status_xfer = dma_readl(dw, RAW.XFER);  in dwc_scan_descriptors()
358 		dma_writel(dw, CLEAR.XFER, dwc->mask);  in dwc_scan_descriptors()
394 		dwc_complete_all(dw, dwc);  in dwc_scan_descriptors()
454 	dwc_chan_disable(dw, dwc);  in dwc_scan_descriptors()
466 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)  in dwc_handle_error()  argument
472 	dwc_scan_descriptors(dw, dwc);  in dwc_handle_error()
486 	dma_writel(dw, CLEAR.ERROR, dwc->mask);  in dwc_handle_error()
526 static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,  in dwc_handle_cyclic()  argument
537 		dma_writel(dw, CLEAR.BLOCK, dwc->mask);  in dwc_handle_cyclic()
562 		dwc_chan_disable(dw, dwc);  in dwc_handle_cyclic()
569 		dma_writel(dw, CLEAR.BLOCK, dwc->mask);  in dwc_handle_cyclic()
570 		dma_writel(dw, CLEAR.ERROR, dwc->mask);  in dwc_handle_cyclic()
571 		dma_writel(dw, CLEAR.XFER, dwc->mask);  in dwc_handle_cyclic()
580 	channel_set_bit(dw, MASK.BLOCK, dwc->mask);  in dwc_handle_cyclic()
587 	struct dw_dma *dw = (struct dw_dma *)data;  in dw_dma_tasklet()  local
594 	status_block = dma_readl(dw, RAW.BLOCK);  in dw_dma_tasklet()
595 	status_xfer = dma_readl(dw, RAW.XFER);  in dw_dma_tasklet()
596 	status_err = dma_readl(dw, RAW.ERROR);  in dw_dma_tasklet()
598 	dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);  in dw_dma_tasklet()
600 	for (i = 0; i < dw->dma.chancnt; i++) {  in dw_dma_tasklet()
601 		dwc = &dw->chan[i];  in dw_dma_tasklet()
603 			dwc_handle_cyclic(dw, dwc, status_block, status_err,  in dw_dma_tasklet()
606 			dwc_handle_error(dw, dwc);  in dw_dma_tasklet()
608 			dwc_scan_descriptors(dw, dwc);  in dw_dma_tasklet()
612 	channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);  in dw_dma_tasklet()
613 	channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);  in dw_dma_tasklet()
618 	struct dw_dma *dw = dev_id;  in dw_dma_interrupt()  local
619 	u32 status = dma_readl(dw, STATUS_INT);  in dw_dma_interrupt()
621 	dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__, status);  in dw_dma_interrupt()
624 	if (!status || !dw->in_use)  in dw_dma_interrupt()
631 	channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);  in dw_dma_interrupt()
632 	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);  in dw_dma_interrupt()
633 	channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);  in dw_dma_interrupt()
635 	status = dma_readl(dw, STATUS_INT);  in dw_dma_interrupt()
637 		dev_err(dw->dma.dev,  in dw_dma_interrupt()
642 		channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);  in dw_dma_interrupt()
643 		channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1);  in dw_dma_interrupt()
644 		channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);  in dw_dma_interrupt()
645 		channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);  in dw_dma_interrupt()
646 		channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);  in dw_dma_interrupt()
649 	tasklet_schedule(&dw->tasklet);  in dw_dma_interrupt()
685 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_prep_dma_memcpy()  local
707 	data_width = min_t(unsigned int, dw->data_width[dwc->src_master],  in dwc_prep_dma_memcpy()
708 			   dw->data_width[dwc->dst_master]);  in dwc_prep_dma_memcpy()
766 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_prep_slave_sg()  local
800 		data_width = dw->data_width[dwc->src_master];  in dwc_prep_slave_sg()
857 		data_width = dw->data_width[dwc->dst_master];  in dwc_prep_slave_sg()
1025 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_terminate_all()  local
1034 	dwc_chan_disable(dw, dwc);  in dwc_terminate_all()
1103 static void dw_dma_off(struct dw_dma *dw)  in dw_dma_off()  argument
1107 	dma_writel(dw, CFG, 0);  in dw_dma_off()
1109 	channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);  in dw_dma_off()
1110 	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);  in dw_dma_off()
1111 	channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);  in dw_dma_off()
1112 	channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);  in dw_dma_off()
1113 	channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);  in dw_dma_off()
1115 	while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)  in dw_dma_off()
1118 	for (i = 0; i < dw->dma.chancnt; i++)  in dw_dma_off()
1119 		dw->chan[i].initialized = false;  in dw_dma_off()
1122 static void dw_dma_on(struct dw_dma *dw)  in dw_dma_on()  argument
1124 	dma_writel(dw, CFG, DW_CFG_DMA_EN);  in dw_dma_on()
1130 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_alloc_chan_resources()  local
1138 	if (dma_readl(dw, CH_EN) & dwc->mask) {  in dwc_alloc_chan_resources()
1160 	if (!dw->in_use)  in dwc_alloc_chan_resources()
1161 		dw_dma_on(dw);  in dwc_alloc_chan_resources()
1162 	dw->in_use |= dwc->mask;  in dwc_alloc_chan_resources()
1171 		desc = dma_pool_alloc(dw->desc_pool, GFP_ATOMIC, &phys);  in dwc_alloc_chan_resources()
1204 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dwc_free_chan_resources()  local
1231 	channel_clear_bit(dw, MASK.XFER, dwc->mask);  in dwc_free_chan_resources()
1232 	channel_clear_bit(dw, MASK.BLOCK, dwc->mask);  in dwc_free_chan_resources()
1233 	channel_clear_bit(dw, MASK.ERROR, dwc->mask);  in dwc_free_chan_resources()
1238 	dw->in_use &= ~dwc->mask;  in dwc_free_chan_resources()
1239 	if (!dw->in_use)  in dwc_free_chan_resources()
1240 		dw_dma_off(dw);  in dwc_free_chan_resources()
1244 		dma_pool_free(dw->desc_pool, desc, desc->txd.phys);  in dwc_free_chan_resources()
1262 	struct dw_dma		*dw = to_dw_dma(chan->device);  in dw_dma_cyclic_start()  local
1273 	channel_set_bit(dw, MASK.BLOCK, dwc->mask);  in dw_dma_cyclic_start()
1292 	struct dw_dma		*dw = to_dw_dma(dwc->chan.device);  in dw_dma_cyclic_stop()  local
1297 	dwc_chan_disable(dw, dwc);  in dw_dma_cyclic_stop()
1467 	struct dw_dma		*dw = to_dw_dma(dwc->chan.device);  in dw_dma_cyclic_free()  local
1479 	dwc_chan_disable(dw, dwc);  in dw_dma_cyclic_free()
1481 	dma_writel(dw, CLEAR.BLOCK, dwc->mask);  in dw_dma_cyclic_free()
1482 	dma_writel(dw, CLEAR.ERROR, dwc->mask);  in dw_dma_cyclic_free()
1483 	dma_writel(dw, CLEAR.XFER, dwc->mask);  in dw_dma_cyclic_free()
1501 	struct dw_dma		*dw;  in dw_dma_probe()  local
1508 	dw = devm_kzalloc(chip->dev, sizeof(*dw), GFP_KERNEL);  in dw_dma_probe()
1509 	if (!dw)  in dw_dma_probe()
1512 	dw->regs = chip->regs;  in dw_dma_probe()
1513 	chip->dw = dw;  in dw_dma_probe()
1540 		max_blk_size = dma_readl(dw, MAX_BLK_SIZE);  in dw_dma_probe()
1552 	dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan),  in dw_dma_probe()
1554 	if (!dw->chan) {  in dw_dma_probe()
1560 	dw->nr_masters = pdata->nr_masters;  in dw_dma_probe()
1561 	for (i = 0; i < dw->nr_masters; i++)  in dw_dma_probe()
1562 		dw->data_width[i] = pdata->data_width[i];  in dw_dma_probe()
1565 	dw->all_chan_mask = (1 << pdata->nr_channels) - 1;  in dw_dma_probe()
1568 	dw_dma_off(dw);  in dw_dma_probe()
1571 	dw->desc_pool = dmam_pool_create("dw_dmac_desc_pool", chip->dev,  in dw_dma_probe()
1573 	if (!dw->desc_pool) {  in dw_dma_probe()
1579 	tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);  in dw_dma_probe()
1582 			  "dw_dmac", dw);  in dw_dma_probe()
1586 	INIT_LIST_HEAD(&dw->dma.channels);  in dw_dma_probe()
1588 		struct dw_dma_chan	*dwc = &dw->chan[i];  in dw_dma_probe()
1590 		dwc->chan.device = &dw->dma;  in dw_dma_probe()
1594 					&dw->dma.channels);  in dw_dma_probe()
1596 			list_add(&dwc->chan.device_node, &dw->dma.channels);  in dw_dma_probe()
1604 		dwc->ch_regs = &__dw_regs(dw)->CHAN[i];  in dw_dma_probe()
1612 		channel_clear_bit(dw, CH_EN, dwc->mask);  in dw_dma_probe()
1648 	dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);  in dw_dma_probe()
1649 	dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);  in dw_dma_probe()
1650 	dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);  in dw_dma_probe()
1651 	dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);  in dw_dma_probe()
1652 	dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);  in dw_dma_probe()
1655 	dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);  in dw_dma_probe()
1657 		dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);  in dw_dma_probe()
1659 		dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);  in dw_dma_probe()
1661 	dw->dma.dev = chip->dev;  in dw_dma_probe()
1662 	dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;  in dw_dma_probe()
1663 	dw->dma.device_free_chan_resources = dwc_free_chan_resources;  in dw_dma_probe()
1665 	dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;  in dw_dma_probe()
1666 	dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;  in dw_dma_probe()
1668 	dw->dma.device_config = dwc_config;  in dw_dma_probe()
1669 	dw->dma.device_pause = dwc_pause;  in dw_dma_probe()
1670 	dw->dma.device_resume = dwc_resume;  in dw_dma_probe()
1671 	dw->dma.device_terminate_all = dwc_terminate_all;  in dw_dma_probe()
1673 	dw->dma.device_tx_status = dwc_tx_status;  in dw_dma_probe()
1674 	dw->dma.device_issue_pending = dwc_issue_pending;  in dw_dma_probe()
1677 	dw->dma.src_addr_widths = DW_DMA_BUSWIDTHS;  in dw_dma_probe()
1678 	dw->dma.dst_addr_widths = DW_DMA_BUSWIDTHS;  in dw_dma_probe()
1679 	dw->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |  in dw_dma_probe()
1681 	dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in dw_dma_probe()
1683 	err = dma_async_device_register(&dw->dma);  in dw_dma_probe()
1695 	free_irq(chip->irq, dw);  in dw_dma_probe()
1704 	struct dw_dma		*dw = chip->dw;  in dw_dma_remove()  local
1709 	dw_dma_off(dw);  in dw_dma_remove()
1710 	dma_async_device_unregister(&dw->dma);  in dw_dma_remove()
1712 	free_irq(chip->irq, dw);  in dw_dma_remove()
1713 	tasklet_kill(&dw->tasklet);  in dw_dma_remove()
1715 	list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,  in dw_dma_remove()
1718 		channel_clear_bit(dw, CH_EN, dwc->mask);  in dw_dma_remove()
1728 	struct dw_dma *dw = chip->dw;  in dw_dma_disable()  local
1730 	dw_dma_off(dw);  in dw_dma_disable()
1737 	struct dw_dma *dw = chip->dw;  in dw_dma_enable()  local
1739 	dw_dma_on(dw);  in dw_dma_enable()