303 	struct sdma_engine		*sdma;  member
521 static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)  in chnenbl_ofs()  argument
523 	u32 chnenbl0 = sdma->drvdata->chnenbl0;  in chnenbl_ofs()
530 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_config_ownership()  local
537 	evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);  in sdma_config_ownership()
538 	mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);  in sdma_config_ownership()
539 	dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);  in sdma_config_ownership()
556 	writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);  in sdma_config_ownership()
557 	writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);  in sdma_config_ownership()
558 	writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);  in sdma_config_ownership()
563 static void sdma_enable_channel(struct sdma_engine *sdma, int channel)  in sdma_enable_channel()  argument
565 	writel(BIT(channel), sdma->regs + SDMA_H_START);  in sdma_enable_channel()
571 static int sdma_run_channel0(struct sdma_engine *sdma)  in sdma_run_channel0()  argument
576 	sdma_enable_channel(sdma, 0);  in sdma_run_channel0()
578 	while (!(ret = readl_relaxed(sdma->regs + SDMA_H_INTR) & 1)) {  in sdma_run_channel0()
586 		writel_relaxed(ret, sdma->regs + SDMA_H_INTR);  in sdma_run_channel0()
588 		dev_err(sdma->dev, "Timeout waiting for CH0 ready\n");  in sdma_run_channel0()
592 	if (readl(sdma->regs + SDMA_H_CONFIG) == 0)  in sdma_run_channel0()
593 		writel_relaxed(SDMA_H_CONFIG_CSM, sdma->regs + SDMA_H_CONFIG);  in sdma_run_channel0()
598 static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,  in sdma_load_script()  argument
601 	struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd;  in sdma_load_script()
614 	spin_lock_irqsave(&sdma->channel_0_lock, flags);  in sdma_load_script()
624 	ret = sdma_run_channel0(sdma);  in sdma_load_script()
626 	spin_unlock_irqrestore(&sdma->channel_0_lock, flags);  in sdma_load_script()
635 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_enable()  local
638 	u32 chnenbl = chnenbl_ofs(sdma, event);  in sdma_event_enable()
640 	val = readl_relaxed(sdma->regs + chnenbl);  in sdma_event_enable()
642 	writel_relaxed(val, sdma->regs + chnenbl);  in sdma_event_enable()
647 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_disable()  local
649 	u32 chnenbl = chnenbl_ofs(sdma, event);  in sdma_event_disable()
652 	val = readl_relaxed(sdma->regs + chnenbl);  in sdma_event_disable()
654 	writel_relaxed(val, sdma->regs + chnenbl);  in sdma_event_disable()
726 	struct sdma_engine *sdma = dev_id;  in sdma_int_handler()  local
729 	stat = readl_relaxed(sdma->regs + SDMA_H_INTR);  in sdma_int_handler()
732 	writel_relaxed(stat, sdma->regs + SDMA_H_INTR);  in sdma_int_handler()
736 		struct sdma_channel *sdmac = &sdma->channel[channel];  in sdma_int_handler()
755 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_get_pc()  local
769 		emi_2_emi = sdma->script_addrs->ap_2_ap_addr;  in sdma_get_pc()
772 		emi_2_per = sdma->script_addrs->bp_2_ap_addr;  in sdma_get_pc()
773 		per_2_emi = sdma->script_addrs->ap_2_bp_addr;  in sdma_get_pc()
776 		per_2_emi = sdma->script_addrs->firi_2_mcu_addr;  in sdma_get_pc()
777 		emi_2_per = sdma->script_addrs->mcu_2_firi_addr;  in sdma_get_pc()
780 		per_2_emi = sdma->script_addrs->uart_2_mcu_addr;  in sdma_get_pc()
781 		emi_2_per = sdma->script_addrs->mcu_2_app_addr;  in sdma_get_pc()
784 		per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;  in sdma_get_pc()
785 		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;  in sdma_get_pc()
788 		per_2_emi = sdma->script_addrs->ata_2_mcu_addr;  in sdma_get_pc()
789 		emi_2_per = sdma->script_addrs->mcu_2_ata_addr;  in sdma_get_pc()
795 		per_2_emi = sdma->script_addrs->app_2_mcu_addr;  in sdma_get_pc()
796 		emi_2_per = sdma->script_addrs->mcu_2_app_addr;  in sdma_get_pc()
799 		per_2_emi = sdma->script_addrs->ssish_2_mcu_addr;  in sdma_get_pc()
800 		emi_2_per = sdma->script_addrs->mcu_2_ssish_addr;  in sdma_get_pc()
808 		per_2_emi = sdma->script_addrs->shp_2_mcu_addr;  in sdma_get_pc()
809 		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;  in sdma_get_pc()
812 		per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;  in sdma_get_pc()
813 		emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;  in sdma_get_pc()
814 		per_2_per = sdma->script_addrs->per_2_per_addr;  in sdma_get_pc()
817 		per_2_emi = sdma->script_addrs->shp_2_mcu_addr;  in sdma_get_pc()
818 		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;  in sdma_get_pc()
819 		per_2_per = sdma->script_addrs->per_2_per_addr;  in sdma_get_pc()
822 		per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;  in sdma_get_pc()
823 		emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;  in sdma_get_pc()
826 		per_2_emi = sdma->script_addrs->dptc_dvfs_addr;  in sdma_get_pc()
829 		per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;  in sdma_get_pc()
830 		emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;  in sdma_get_pc()
833 		emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;  in sdma_get_pc()
846 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_load_context()  local
849 	struct sdma_context_data *context = sdma->context;  in sdma_load_context()
850 	struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd;  in sdma_load_context()
864 	dev_dbg(sdma->dev, "load_address = %d\n", load_address);  in sdma_load_context()
865 	dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);  in sdma_load_context()
866 	dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);  in sdma_load_context()
867 	dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);  in sdma_load_context()
868 	dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);  in sdma_load_context()
869 	dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);  in sdma_load_context()
871 	spin_lock_irqsave(&sdma->channel_0_lock, flags);  in sdma_load_context()
888 	bd0->buffer_addr = sdma->context_phys;  in sdma_load_context()
890 	ret = sdma_run_channel0(sdma);  in sdma_load_context()
892 	spin_unlock_irqrestore(&sdma->channel_0_lock, flags);  in sdma_load_context()
905 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_disable_channel()  local
908 	writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);  in sdma_disable_channel()
916 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_set_watermarklevel_for_p2p()  local
943 	if (sdmac->per_address2 >= sdma->spba_start_addr &&  in sdma_set_watermarklevel_for_p2p()
944 			sdmac->per_address2 <= sdma->spba_end_addr)  in sdma_set_watermarklevel_for_p2p()
947 	if (sdmac->per_address >= sdma->spba_start_addr &&  in sdma_set_watermarklevel_for_p2p()
948 			sdmac->per_address <= sdma->spba_end_addr)  in sdma_set_watermarklevel_for_p2p()
967 		if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)  in sdma_config_channel()
973 		if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)  in sdma_config_channel()
1019 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_set_channel_priority()  local
1027 	writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);  in sdma_set_channel_priority()
1034 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_request_channel()  local
1045 	sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys;  in sdma_request_channel()
1046 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;  in sdma_request_channel()
1096 	ret = clk_enable(sdmac->sdma->clk_ipg);  in sdma_alloc_chan_resources()
1099 	ret = clk_enable(sdmac->sdma->clk_ahb);  in sdma_alloc_chan_resources()
1119 	clk_disable(sdmac->sdma->clk_ahb);  in sdma_alloc_chan_resources()
1121 	clk_disable(sdmac->sdma->clk_ipg);  in sdma_alloc_chan_resources()
1128 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_free_chan_resources()  local
1144 	clk_disable(sdma->clk_ipg);  in sdma_free_chan_resources()
1145 	clk_disable(sdma->clk_ahb);  in sdma_free_chan_resources()
1154 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_slave_sg()  local
1167 	dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",  in sdma_prep_slave_sg()
1176 		dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n",  in sdma_prep_slave_sg()
1192 			dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",  in sdma_prep_slave_sg()
1232 		dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",  in sdma_prep_slave_sg()
1241 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;  in sdma_prep_slave_sg()
1255 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_dma_cyclic()  local
1260 	dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);  in sdma_prep_dma_cyclic()
1277 		dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n",  in sdma_prep_dma_cyclic()
1283 		dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %d > %d\n",  in sdma_prep_dma_cyclic()
1307 		dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",  in sdma_prep_dma_cyclic()
1321 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;  in sdma_prep_dma_cyclic()
1378 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_issue_pending()  local
1381 		sdma_enable_channel(sdma, sdmac->channel);  in sdma_issue_pending()
1388 static void sdma_add_scripts(struct sdma_engine *sdma,  in sdma_add_scripts()  argument
1392 	s32 *saddr_arr = (u32 *)sdma->script_addrs;  in sdma_add_scripts()
1396 	if (!sdma->script_number)  in sdma_add_scripts()
1397 		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;  in sdma_add_scripts()
1399 	for (i = 0; i < sdma->script_number; i++)  in sdma_add_scripts()
1406 	struct sdma_engine *sdma = context;  in sdma_load_firmware()  local
1412 		dev_info(sdma->dev, "external firmware not found, using ROM firmware\n");  in sdma_load_firmware()
1428 		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;  in sdma_load_firmware()
1431 		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2;  in sdma_load_firmware()
1434 		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3;  in sdma_load_firmware()
1437 		dev_err(sdma->dev, "unknown firmware version\n");  in sdma_load_firmware()
1444 	clk_enable(sdma->clk_ipg);  in sdma_load_firmware()
1445 	clk_enable(sdma->clk_ahb);  in sdma_load_firmware()
1447 	sdma_load_script(sdma, ram_code,  in sdma_load_firmware()
1450 	clk_disable(sdma->clk_ipg);  in sdma_load_firmware()
1451 	clk_disable(sdma->clk_ahb);  in sdma_load_firmware()
1453 	sdma_add_scripts(sdma, addr);  in sdma_load_firmware()
1455 	dev_info(sdma->dev, "loaded firmware %d.%d\n",  in sdma_load_firmware()
1465 static int sdma_event_remap(struct sdma_engine *sdma)  in sdma_event_remap()  argument
1467 	struct device_node *np = sdma->dev->of_node;  in sdma_event_remap()
1481 		dev_dbg(sdma->dev, "no event needs to be remapped\n");  in sdma_event_remap()
1484 		dev_err(sdma->dev, "the property %s must modulo %d\n",  in sdma_event_remap()
1492 		dev_err(sdma->dev, "failed to get gpr regmap\n");  in sdma_event_remap()
1500 			dev_err(sdma->dev, "failed to read property %s index %d\n",  in sdma_event_remap()
1507 			dev_err(sdma->dev, "failed to read property %s index %d\n",  in sdma_event_remap()
1514 			dev_err(sdma->dev, "failed to read property %s index %d\n",  in sdma_event_remap()
1529 static int sdma_get_firmware(struct sdma_engine *sdma,  in sdma_get_firmware()  argument
1535 			FW_ACTION_HOTPLUG, fw_name, sdma->dev,  in sdma_get_firmware()
1536 			GFP_KERNEL, sdma, sdma_load_firmware);  in sdma_get_firmware()
1541 static int sdma_init(struct sdma_engine *sdma)  in sdma_init()  argument
1546 	ret = clk_enable(sdma->clk_ipg);  in sdma_init()
1549 	ret = clk_enable(sdma->clk_ahb);  in sdma_init()
1554 	writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);  in sdma_init()
1556 	sdma->channel_control = dma_alloc_coherent(NULL,  in sdma_init()
1561 	if (!sdma->channel_control) {  in sdma_init()
1566 	sdma->context = (void *)sdma->channel_control +  in sdma_init()
1568 	sdma->context_phys = ccb_phys +  in sdma_init()
1572 	memset(sdma->channel_control, 0,  in sdma_init()
1576 	for (i = 0; i < sdma->drvdata->num_events; i++)  in sdma_init()
1577 		writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));  in sdma_init()
1581 		writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);  in sdma_init()
1583 	ret = sdma_request_channel(&sdma->channel[0]);  in sdma_init()
1587 	sdma_config_ownership(&sdma->channel[0], false, true, false);  in sdma_init()
1590 	writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);  in sdma_init()
1594 	writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);  in sdma_init()
1596 	writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR);  in sdma_init()
1599 	sdma_set_channel_priority(&sdma->channel[0], 7);  in sdma_init()
1601 	clk_disable(sdma->clk_ipg);  in sdma_init()
1602 	clk_disable(sdma->clk_ahb);  in sdma_init()
1607 	clk_disable(sdma->clk_ahb);  in sdma_init()
1609 	clk_disable(sdma->clk_ipg);  in sdma_init()
1610 	dev_err(sdma->dev, "initialisation failed with %d\n", ret);  in sdma_init()
1631 	struct sdma_engine *sdma = ofdma->of_dma_data;  in sdma_xlate()  local
1632 	dma_cap_mask_t mask = sdma->dma_device.cap_mask;  in sdma_xlate()
1666 	struct sdma_engine *sdma;  in sdma_probe()  local
1684 	sdma = devm_kzalloc(&pdev->dev, sizeof(*sdma), GFP_KERNEL);  in sdma_probe()
1685 	if (!sdma)  in sdma_probe()
1688 	spin_lock_init(&sdma->channel_0_lock);  in sdma_probe()
1690 	sdma->dev = &pdev->dev;  in sdma_probe()
1691 	sdma->drvdata = drvdata;  in sdma_probe()
1698 	sdma->regs = devm_ioremap_resource(&pdev->dev, iores);  in sdma_probe()
1699 	if (IS_ERR(sdma->regs))  in sdma_probe()
1700 		return PTR_ERR(sdma->regs);  in sdma_probe()
1702 	sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg");  in sdma_probe()
1703 	if (IS_ERR(sdma->clk_ipg))  in sdma_probe()
1704 		return PTR_ERR(sdma->clk_ipg);  in sdma_probe()
1706 	sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb");  in sdma_probe()
1707 	if (IS_ERR(sdma->clk_ahb))  in sdma_probe()
1708 		return PTR_ERR(sdma->clk_ahb);  in sdma_probe()
1710 	clk_prepare(sdma->clk_ipg);  in sdma_probe()
1711 	clk_prepare(sdma->clk_ahb);  in sdma_probe()
1714 			       sdma);  in sdma_probe()
1718 	sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL);  in sdma_probe()
1719 	if (!sdma->script_addrs)  in sdma_probe()
1723 	saddr_arr = (s32 *)sdma->script_addrs;  in sdma_probe()
1727 	dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);  in sdma_probe()
1728 	dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);  in sdma_probe()
1730 	INIT_LIST_HEAD(&sdma->dma_device.channels);  in sdma_probe()
1733 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_probe()
1735 		sdmac->sdma = sdma;  in sdma_probe()
1738 		sdmac->chan.device = &sdma->dma_device;  in sdma_probe()
1751 					&sdma->dma_device.channels);  in sdma_probe()
1754 	ret = sdma_init(sdma);  in sdma_probe()
1758 	ret = sdma_event_remap(sdma);  in sdma_probe()
1762 	if (sdma->drvdata->script_addrs)  in sdma_probe()
1763 		sdma_add_scripts(sdma, sdma->drvdata->script_addrs);  in sdma_probe()
1765 		sdma_add_scripts(sdma, pdata->script_addrs);  in sdma_probe()
1768 		ret = sdma_get_firmware(sdma, pdata->fw_name);  in sdma_probe()
1782 			ret = sdma_get_firmware(sdma, fw_name);  in sdma_probe()
1788 	sdma->dma_device.dev = &pdev->dev;  in sdma_probe()
1790 	sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources;  in sdma_probe()
1791 	sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources;  in sdma_probe()
1792 	sdma->dma_device.device_tx_status = sdma_tx_status;  in sdma_probe()
1793 	sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;  in sdma_probe()
1794 	sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;  in sdma_probe()
1795 	sdma->dma_device.device_config = sdma_config;  in sdma_probe()
1796 	sdma->dma_device.device_terminate_all = sdma_disable_channel;  in sdma_probe()
1797 	sdma->dma_device.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);  in sdma_probe()
1798 	sdma->dma_device.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);  in sdma_probe()
1799 	sdma->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in sdma_probe()
1800 	sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in sdma_probe()
1801 	sdma->dma_device.device_issue_pending = sdma_issue_pending;  in sdma_probe()
1802 	sdma->dma_device.dev->dma_parms = &sdma->dma_parms;  in sdma_probe()
1803 	dma_set_max_seg_size(sdma->dma_device.dev, 65535);  in sdma_probe()
1805 	platform_set_drvdata(pdev, sdma);  in sdma_probe()
1807 	ret = dma_async_device_register(&sdma->dma_device);  in sdma_probe()
1814 		ret = of_dma_controller_register(np, sdma_xlate, sdma);  in sdma_probe()
1823 			sdma->spba_start_addr = spba_res.start;  in sdma_probe()
1824 			sdma->spba_end_addr = spba_res.end;  in sdma_probe()
1832 	dma_async_device_unregister(&sdma->dma_device);  in sdma_probe()
1834 	kfree(sdma->script_addrs);  in sdma_probe()
1840 	struct sdma_engine *sdma = platform_get_drvdata(pdev);  in sdma_remove()  local
1843 	dma_async_device_unregister(&sdma->dma_device);  in sdma_remove()
1844 	kfree(sdma->script_addrs);  in sdma_remove()
1847 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_remove()