527 static int sdma_config_ownership(struct sdma_channel *sdmac,  in sdma_config_ownership()  argument
530 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_config_ownership()
531 	int channel = sdmac->channel;  in sdma_config_ownership()
633 static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)  in sdma_event_enable()  argument
635 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_enable()
636 	int channel = sdmac->channel;  in sdma_event_enable()
645 static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)  in sdma_event_disable()  argument
647 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_disable()
648 	int channel = sdmac->channel;  in sdma_event_disable()
657 static void sdma_handle_channel_loop(struct sdma_channel *sdmac)  in sdma_handle_channel_loop()  argument
659 	if (sdmac->desc.callback)  in sdma_handle_channel_loop()
660 		sdmac->desc.callback(sdmac->desc.callback_param);  in sdma_handle_channel_loop()
663 static void sdma_update_channel_loop(struct sdma_channel *sdmac)  in sdma_update_channel_loop()  argument
672 		bd = &sdmac->bd[sdmac->buf_tail];  in sdma_update_channel_loop()
678 			sdmac->status = DMA_ERROR;  in sdma_update_channel_loop()
681 		sdmac->buf_tail++;  in sdma_update_channel_loop()
682 		sdmac->buf_tail %= sdmac->num_bd;  in sdma_update_channel_loop()
686 static void mxc_sdma_handle_channel_normal(struct sdma_channel *sdmac)  in mxc_sdma_handle_channel_normal()  argument
691 	sdmac->chn_real_count = 0;  in mxc_sdma_handle_channel_normal()
696 	for (i = 0; i < sdmac->num_bd; i++) {  in mxc_sdma_handle_channel_normal()
697 		bd = &sdmac->bd[i];  in mxc_sdma_handle_channel_normal()
701 		 sdmac->chn_real_count += bd->mode.count;  in mxc_sdma_handle_channel_normal()
705 		sdmac->status = DMA_ERROR;  in mxc_sdma_handle_channel_normal()
707 		sdmac->status = DMA_COMPLETE;  in mxc_sdma_handle_channel_normal()
709 	dma_cookie_complete(&sdmac->desc);  in mxc_sdma_handle_channel_normal()
710 	if (sdmac->desc.callback)  in mxc_sdma_handle_channel_normal()
711 		sdmac->desc.callback(sdmac->desc.callback_param);  in mxc_sdma_handle_channel_normal()
716 	struct sdma_channel *sdmac = (struct sdma_channel *) data;  in sdma_tasklet()  local
718 	if (sdmac->flags & IMX_DMA_SG_LOOP)  in sdma_tasklet()
719 		sdma_handle_channel_loop(sdmac);  in sdma_tasklet()
721 		mxc_sdma_handle_channel_normal(sdmac);  in sdma_tasklet()
736 		struct sdma_channel *sdmac = &sdma->channel[channel];  in sdma_int_handler()  local
738 		if (sdmac->flags & IMX_DMA_SG_LOOP)  in sdma_int_handler()
739 			sdma_update_channel_loop(sdmac);  in sdma_int_handler()
741 		tasklet_schedule(&sdmac->tasklet);  in sdma_int_handler()
752 static void sdma_get_pc(struct sdma_channel *sdmac,  in sdma_get_pc()  argument
755 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_get_pc()
763 	sdmac->pc_from_device = 0;  in sdma_get_pc()
764 	sdmac->pc_to_device = 0;  in sdma_get_pc()
765 	sdmac->device_to_device = 0;  in sdma_get_pc()
839 	sdmac->pc_from_device = per_2_emi;  in sdma_get_pc()
840 	sdmac->pc_to_device = emi_2_per;  in sdma_get_pc()
841 	sdmac->device_to_device = per_2_per;  in sdma_get_pc()
844 static int sdma_load_context(struct sdma_channel *sdmac)  in sdma_load_context()  argument
846 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_load_context()
847 	int channel = sdmac->channel;  in sdma_load_context()
854 	if (sdmac->direction == DMA_DEV_TO_MEM)  in sdma_load_context()
855 		load_address = sdmac->pc_from_device;  in sdma_load_context()
856 	else if (sdmac->direction == DMA_DEV_TO_DEV)  in sdma_load_context()
857 		load_address = sdmac->device_to_device;  in sdma_load_context()
859 		load_address = sdmac->pc_to_device;  in sdma_load_context()
865 	dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);  in sdma_load_context()
866 	dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);  in sdma_load_context()
867 	dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);  in sdma_load_context()
868 	dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);  in sdma_load_context()
869 	dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);  in sdma_load_context()
879 	context->gReg[0] = sdmac->event_mask[1];  in sdma_load_context()
880 	context->gReg[1] = sdmac->event_mask[0];  in sdma_load_context()
881 	context->gReg[2] = sdmac->per_addr;  in sdma_load_context()
882 	context->gReg[6] = sdmac->shp_addr;  in sdma_load_context()
883 	context->gReg[7] = sdmac->watermark_level;  in sdma_load_context()
904 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_disable_channel()  local
905 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_disable_channel()
906 	int channel = sdmac->channel;  in sdma_disable_channel()
909 	sdmac->status = DMA_ERROR;  in sdma_disable_channel()
914 static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)  in sdma_set_watermarklevel_for_p2p()  argument
916 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_set_watermarklevel_for_p2p()
918 	int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;  in sdma_set_watermarklevel_for_p2p()
919 	int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;  in sdma_set_watermarklevel_for_p2p()
921 	set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);  in sdma_set_watermarklevel_for_p2p()
922 	set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);  in sdma_set_watermarklevel_for_p2p()
924 	if (sdmac->event_id0 > 31)  in sdma_set_watermarklevel_for_p2p()
925 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;  in sdma_set_watermarklevel_for_p2p()
927 	if (sdmac->event_id1 > 31)  in sdma_set_watermarklevel_for_p2p()
928 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;  in sdma_set_watermarklevel_for_p2p()
936 		sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |  in sdma_set_watermarklevel_for_p2p()
938 		sdmac->watermark_level |= hwml;  in sdma_set_watermarklevel_for_p2p()
939 		sdmac->watermark_level |= lwml << 16;  in sdma_set_watermarklevel_for_p2p()
940 		swap(sdmac->event_mask[0], sdmac->event_mask[1]);  in sdma_set_watermarklevel_for_p2p()
943 	if (sdmac->per_address2 >= sdma->spba_start_addr &&  in sdma_set_watermarklevel_for_p2p()
944 			sdmac->per_address2 <= sdma->spba_end_addr)  in sdma_set_watermarklevel_for_p2p()
945 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;  in sdma_set_watermarklevel_for_p2p()
947 	if (sdmac->per_address >= sdma->spba_start_addr &&  in sdma_set_watermarklevel_for_p2p()
948 			sdmac->per_address <= sdma->spba_end_addr)  in sdma_set_watermarklevel_for_p2p()
949 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;  in sdma_set_watermarklevel_for_p2p()
951 	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;  in sdma_set_watermarklevel_for_p2p()
956 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_config_channel()  local
961 	sdmac->event_mask[0] = 0;  in sdma_config_channel()
962 	sdmac->event_mask[1] = 0;  in sdma_config_channel()
963 	sdmac->shp_addr = 0;  in sdma_config_channel()
964 	sdmac->per_addr = 0;  in sdma_config_channel()
966 	if (sdmac->event_id0) {  in sdma_config_channel()
967 		if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)  in sdma_config_channel()
969 		sdma_event_enable(sdmac, sdmac->event_id0);  in sdma_config_channel()
972 	if (sdmac->event_id1) {  in sdma_config_channel()
973 		if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)  in sdma_config_channel()
975 		sdma_event_enable(sdmac, sdmac->event_id1);  in sdma_config_channel()
978 	switch (sdmac->peripheral_type) {  in sdma_config_channel()
980 		sdma_config_ownership(sdmac, false, true, true);  in sdma_config_channel()
983 		sdma_config_ownership(sdmac, false, true, false);  in sdma_config_channel()
986 		sdma_config_ownership(sdmac, true, true, false);  in sdma_config_channel()
990 	sdma_get_pc(sdmac, sdmac->peripheral_type);  in sdma_config_channel()
992 	if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&  in sdma_config_channel()
993 			(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {  in sdma_config_channel()
995 		if (sdmac->event_id1) {  in sdma_config_channel()
996 			if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||  in sdma_config_channel()
997 			    sdmac->peripheral_type == IMX_DMATYPE_ASRC)  in sdma_config_channel()
998 				sdma_set_watermarklevel_for_p2p(sdmac);  in sdma_config_channel()
1000 			__set_bit(sdmac->event_id0, sdmac->event_mask);  in sdma_config_channel()
1003 		sdmac->watermark_level |= sdmac->watermark_level;  in sdma_config_channel()
1005 		sdmac->shp_addr = sdmac->per_address;  in sdma_config_channel()
1006 		sdmac->per_addr = sdmac->per_address2;  in sdma_config_channel()
1008 		sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */  in sdma_config_channel()
1011 	ret = sdma_load_context(sdmac);  in sdma_config_channel()
1016 static int sdma_set_channel_priority(struct sdma_channel *sdmac,  in sdma_set_channel_priority()  argument
1019 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_set_channel_priority()
1020 	int channel = sdmac->channel;  in sdma_set_channel_priority()
1032 static int sdma_request_channel(struct sdma_channel *sdmac)  in sdma_request_channel()  argument
1034 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_request_channel()
1035 	int channel = sdmac->channel;  in sdma_request_channel()
1038 	sdmac->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys,  in sdma_request_channel()
1040 	if (!sdmac->bd) {  in sdma_request_channel()
1045 	sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys;  in sdma_request_channel()
1046 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;  in sdma_request_channel()
1048 	sdma_set_channel_priority(sdmac, MXC_SDMA_DEFAULT_PRIORITY);  in sdma_request_channel()
1058 	struct sdma_channel *sdmac = to_sdma_chan(tx->chan);  in sdma_tx_submit()  local
1061 	spin_lock_irqsave(&sdmac->lock, flags);  in sdma_tx_submit()
1065 	spin_unlock_irqrestore(&sdmac->lock, flags);  in sdma_tx_submit()
1072 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_alloc_chan_resources()  local
1092 	sdmac->peripheral_type = data->peripheral_type;  in sdma_alloc_chan_resources()
1093 	sdmac->event_id0 = data->dma_request;  in sdma_alloc_chan_resources()
1094 	sdmac->event_id1 = data->dma_request2;  in sdma_alloc_chan_resources()
1096 	ret = clk_enable(sdmac->sdma->clk_ipg);  in sdma_alloc_chan_resources()
1099 	ret = clk_enable(sdmac->sdma->clk_ahb);  in sdma_alloc_chan_resources()
1103 	ret = sdma_request_channel(sdmac);  in sdma_alloc_chan_resources()
1107 	ret = sdma_set_channel_priority(sdmac, prio);  in sdma_alloc_chan_resources()
1111 	dma_async_tx_descriptor_init(&sdmac->desc, chan);  in sdma_alloc_chan_resources()
1112 	sdmac->desc.tx_submit = sdma_tx_submit;  in sdma_alloc_chan_resources()
1114 	sdmac->desc.flags = DMA_CTRL_ACK;  in sdma_alloc_chan_resources()
1119 	clk_disable(sdmac->sdma->clk_ahb);  in sdma_alloc_chan_resources()
1121 	clk_disable(sdmac->sdma->clk_ipg);  in sdma_alloc_chan_resources()
1127 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_free_chan_resources()  local
1128 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_free_chan_resources()
1132 	if (sdmac->event_id0)  in sdma_free_chan_resources()
1133 		sdma_event_disable(sdmac, sdmac->event_id0);  in sdma_free_chan_resources()
1134 	if (sdmac->event_id1)  in sdma_free_chan_resources()
1135 		sdma_event_disable(sdmac, sdmac->event_id1);  in sdma_free_chan_resources()
1137 	sdmac->event_id0 = 0;  in sdma_free_chan_resources()
1138 	sdmac->event_id1 = 0;  in sdma_free_chan_resources()
1140 	sdma_set_channel_priority(sdmac, 0);  in sdma_free_chan_resources()
1142 	dma_free_coherent(NULL, PAGE_SIZE, sdmac->bd, sdmac->bd_phys);  in sdma_free_chan_resources()
1153 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_slave_sg()  local
1154 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_slave_sg()
1156 	int channel = sdmac->channel;  in sdma_prep_slave_sg()
1159 	if (sdmac->status == DMA_IN_PROGRESS)  in sdma_prep_slave_sg()
1161 	sdmac->status = DMA_IN_PROGRESS;  in sdma_prep_slave_sg()
1163 	sdmac->flags = 0;  in sdma_prep_slave_sg()
1165 	sdmac->buf_tail = 0;  in sdma_prep_slave_sg()
1170 	sdmac->direction = direction;  in sdma_prep_slave_sg()
1171 	ret = sdma_load_context(sdmac);  in sdma_prep_slave_sg()
1182 	sdmac->chn_count = 0;  in sdma_prep_slave_sg()
1184 		struct sdma_buffer_descriptor *bd = &sdmac->bd[i];  in sdma_prep_slave_sg()
1199 		sdmac->chn_count += count;  in sdma_prep_slave_sg()
1201 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) {  in sdma_prep_slave_sg()
1206 		switch (sdmac->word_size) {  in sdma_prep_slave_sg()
1240 	sdmac->num_bd = sg_len;  in sdma_prep_slave_sg()
1241 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;  in sdma_prep_slave_sg()
1243 	return &sdmac->desc;  in sdma_prep_slave_sg()
1245 	sdmac->status = DMA_ERROR;  in sdma_prep_slave_sg()
1254 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_dma_cyclic()  local
1255 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_dma_cyclic()
1257 	int channel = sdmac->channel;  in sdma_prep_dma_cyclic()
1262 	if (sdmac->status == DMA_IN_PROGRESS)  in sdma_prep_dma_cyclic()
1265 	sdmac->status = DMA_IN_PROGRESS;  in sdma_prep_dma_cyclic()
1267 	sdmac->buf_tail = 0;  in sdma_prep_dma_cyclic()
1268 	sdmac->period_len = period_len;  in sdma_prep_dma_cyclic()
1270 	sdmac->flags |= IMX_DMA_SG_LOOP;  in sdma_prep_dma_cyclic()
1271 	sdmac->direction = direction;  in sdma_prep_dma_cyclic()
1272 	ret = sdma_load_context(sdmac);  in sdma_prep_dma_cyclic()
1289 		struct sdma_buffer_descriptor *bd = &sdmac->bd[i];  in sdma_prep_dma_cyclic()
1296 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_dma_cyclic()
1298 		if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_dma_cyclic()
1301 			bd->mode.command = sdmac->word_size;  in sdma_prep_dma_cyclic()
1320 	sdmac->num_bd = num_periods;  in sdma_prep_dma_cyclic()
1321 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;  in sdma_prep_dma_cyclic()
1323 	return &sdmac->desc;  in sdma_prep_dma_cyclic()
1325 	sdmac->status = DMA_ERROR;  in sdma_prep_dma_cyclic()
1332 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_config()  local
1335 		sdmac->per_address = dmaengine_cfg->src_addr;  in sdma_config()
1336 		sdmac->watermark_level = dmaengine_cfg->src_maxburst *  in sdma_config()
1338 		sdmac->word_size = dmaengine_cfg->src_addr_width;  in sdma_config()
1340 		sdmac->per_address2 = dmaengine_cfg->src_addr;  in sdma_config()
1341 		sdmac->per_address = dmaengine_cfg->dst_addr;  in sdma_config()
1342 		sdmac->watermark_level = dmaengine_cfg->src_maxburst &  in sdma_config()
1344 		sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &  in sdma_config()
1346 		sdmac->word_size = dmaengine_cfg->dst_addr_width;  in sdma_config()
1348 		sdmac->per_address = dmaengine_cfg->dst_addr;  in sdma_config()
1349 		sdmac->watermark_level = dmaengine_cfg->dst_maxburst *  in sdma_config()
1351 		sdmac->word_size = dmaengine_cfg->dst_addr_width;  in sdma_config()
1353 	sdmac->direction = dmaengine_cfg->direction;  in sdma_config()
1361 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_tx_status()  local
1364 	if (sdmac->flags & IMX_DMA_SG_LOOP)  in sdma_tx_status()
1365 		residue = (sdmac->num_bd - sdmac->buf_tail) * sdmac->period_len;  in sdma_tx_status()
1367 		residue = sdmac->chn_count - sdmac->chn_real_count;  in sdma_tx_status()
1372 	return sdmac->status;  in sdma_tx_status()
1377 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_issue_pending()  local
1378 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_issue_pending()
1380 	if (sdmac->status == DMA_IN_PROGRESS)  in sdma_issue_pending()
1381 		sdma_enable_channel(sdma, sdmac->channel);  in sdma_issue_pending()
1616 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_filter_fn()  local
1622 	sdmac->data = *data;  in sdma_filter_fn()
1623 	chan->private = &sdmac->data;  in sdma_filter_fn()
1733 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_probe()  local
1735 		sdmac->sdma = sdma;  in sdma_probe()
1736 		spin_lock_init(&sdmac->lock);  in sdma_probe()
1738 		sdmac->chan.device = &sdma->dma_device;  in sdma_probe()
1739 		dma_cookie_init(&sdmac->chan);  in sdma_probe()
1740 		sdmac->channel = i;  in sdma_probe()
1742 		tasklet_init(&sdmac->tasklet, sdma_tasklet,  in sdma_probe()
1743 			     (unsigned long) sdmac);  in sdma_probe()
1750 			list_add_tail(&sdmac->chan.device_node,  in sdma_probe()
1847 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_remove()  local
1849 		tasklet_kill(&sdmac->tasklet);  in sdma_remove()