470 static int sdma_config_ownership(struct sdma_channel *sdmac,  in sdma_config_ownership()  argument
473 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_config_ownership()
474 	int channel = sdmac->channel;  in sdma_config_ownership()
576 static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)  in sdma_event_enable()  argument
578 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_enable()
579 	int channel = sdmac->channel;  in sdma_event_enable()
588 static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)  in sdma_event_disable()  argument
590 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_disable()
591 	int channel = sdmac->channel;  in sdma_event_disable()
600 static void sdma_handle_channel_loop(struct sdma_channel *sdmac)  in sdma_handle_channel_loop()  argument
602 	if (sdmac->desc.callback)  in sdma_handle_channel_loop()
603 		sdmac->desc.callback(sdmac->desc.callback_param);  in sdma_handle_channel_loop()
606 static void sdma_update_channel_loop(struct sdma_channel *sdmac)  in sdma_update_channel_loop()  argument
615 		bd = &sdmac->bd[sdmac->buf_tail];  in sdma_update_channel_loop()
621 			sdmac->status = DMA_ERROR;  in sdma_update_channel_loop()
624 		sdmac->buf_tail++;  in sdma_update_channel_loop()
625 		sdmac->buf_tail %= sdmac->num_bd;  in sdma_update_channel_loop()
629 static void mxc_sdma_handle_channel_normal(struct sdma_channel *sdmac)  in mxc_sdma_handle_channel_normal()  argument
634 	sdmac->chn_real_count = 0;  in mxc_sdma_handle_channel_normal()
639 	for (i = 0; i < sdmac->num_bd; i++) {  in mxc_sdma_handle_channel_normal()
640 		bd = &sdmac->bd[i];  in mxc_sdma_handle_channel_normal()
644 		 sdmac->chn_real_count += bd->mode.count;  in mxc_sdma_handle_channel_normal()
648 		sdmac->status = DMA_ERROR;  in mxc_sdma_handle_channel_normal()
650 		sdmac->status = DMA_COMPLETE;  in mxc_sdma_handle_channel_normal()
652 	dma_cookie_complete(&sdmac->desc);  in mxc_sdma_handle_channel_normal()
653 	if (sdmac->desc.callback)  in mxc_sdma_handle_channel_normal()
654 		sdmac->desc.callback(sdmac->desc.callback_param);  in mxc_sdma_handle_channel_normal()
659 	struct sdma_channel *sdmac = (struct sdma_channel *) data;  in sdma_tasklet()  local
661 	if (sdmac->flags & IMX_DMA_SG_LOOP)  in sdma_tasklet()
662 		sdma_handle_channel_loop(sdmac);  in sdma_tasklet()
664 		mxc_sdma_handle_channel_normal(sdmac);  in sdma_tasklet()
679 		struct sdma_channel *sdmac = &sdma->channel[channel];  in sdma_int_handler()  local
681 		if (sdmac->flags & IMX_DMA_SG_LOOP)  in sdma_int_handler()
682 			sdma_update_channel_loop(sdmac);  in sdma_int_handler()
684 		tasklet_schedule(&sdmac->tasklet);  in sdma_int_handler()
695 static void sdma_get_pc(struct sdma_channel *sdmac,  in sdma_get_pc()  argument
698 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_get_pc()
706 	sdmac->pc_from_device = 0;  in sdma_get_pc()
707 	sdmac->pc_to_device = 0;  in sdma_get_pc()
781 	sdmac->pc_from_device = per_2_emi;  in sdma_get_pc()
782 	sdmac->pc_to_device = emi_2_per;  in sdma_get_pc()
785 static int sdma_load_context(struct sdma_channel *sdmac)  in sdma_load_context()  argument
787 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_load_context()
788 	int channel = sdmac->channel;  in sdma_load_context()
795 	if (sdmac->direction == DMA_DEV_TO_MEM) {  in sdma_load_context()
796 		load_address = sdmac->pc_from_device;  in sdma_load_context()
798 		load_address = sdmac->pc_to_device;  in sdma_load_context()
805 	dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);  in sdma_load_context()
806 	dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);  in sdma_load_context()
807 	dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);  in sdma_load_context()
808 	dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);  in sdma_load_context()
809 	dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);  in sdma_load_context()
819 	context->gReg[0] = sdmac->event_mask[1];  in sdma_load_context()
820 	context->gReg[1] = sdmac->event_mask[0];  in sdma_load_context()
821 	context->gReg[2] = sdmac->per_addr;  in sdma_load_context()
822 	context->gReg[6] = sdmac->shp_addr;  in sdma_load_context()
823 	context->gReg[7] = sdmac->watermark_level;  in sdma_load_context()
844 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_disable_channel()  local
845 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_disable_channel()
846 	int channel = sdmac->channel;  in sdma_disable_channel()
849 	sdmac->status = DMA_ERROR;  in sdma_disable_channel()
856 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_config_channel()  local
861 	sdmac->event_mask[0] = 0;  in sdma_config_channel()
862 	sdmac->event_mask[1] = 0;  in sdma_config_channel()
863 	sdmac->shp_addr = 0;  in sdma_config_channel()
864 	sdmac->per_addr = 0;  in sdma_config_channel()
866 	if (sdmac->event_id0) {  in sdma_config_channel()
867 		if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)  in sdma_config_channel()
869 		sdma_event_enable(sdmac, sdmac->event_id0);  in sdma_config_channel()
872 	switch (sdmac->peripheral_type) {  in sdma_config_channel()
874 		sdma_config_ownership(sdmac, false, true, true);  in sdma_config_channel()
877 		sdma_config_ownership(sdmac, false, true, false);  in sdma_config_channel()
880 		sdma_config_ownership(sdmac, true, true, false);  in sdma_config_channel()
884 	sdma_get_pc(sdmac, sdmac->peripheral_type);  in sdma_config_channel()
886 	if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&  in sdma_config_channel()
887 			(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {  in sdma_config_channel()
889 		if (sdmac->event_id1) {  in sdma_config_channel()
890 			sdmac->event_mask[1] = BIT(sdmac->event_id1 % 32);  in sdma_config_channel()
891 			if (sdmac->event_id1 > 31)  in sdma_config_channel()
892 				__set_bit(31, &sdmac->watermark_level);  in sdma_config_channel()
893 			sdmac->event_mask[0] = BIT(sdmac->event_id0 % 32);  in sdma_config_channel()
894 			if (sdmac->event_id0 > 31)  in sdma_config_channel()
895 				__set_bit(30, &sdmac->watermark_level);  in sdma_config_channel()
897 			__set_bit(sdmac->event_id0, sdmac->event_mask);  in sdma_config_channel()
900 		sdmac->watermark_level |= sdmac->watermark_level;  in sdma_config_channel()
902 		sdmac->shp_addr = sdmac->per_address;  in sdma_config_channel()
904 		sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */  in sdma_config_channel()
907 	ret = sdma_load_context(sdmac);  in sdma_config_channel()
912 static int sdma_set_channel_priority(struct sdma_channel *sdmac,  in sdma_set_channel_priority()  argument
915 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_set_channel_priority()
916 	int channel = sdmac->channel;  in sdma_set_channel_priority()
928 static int sdma_request_channel(struct sdma_channel *sdmac)  in sdma_request_channel()  argument
930 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_request_channel()
931 	int channel = sdmac->channel;  in sdma_request_channel()
934 	sdmac->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys,  in sdma_request_channel()
936 	if (!sdmac->bd) {  in sdma_request_channel()
941 	sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys;  in sdma_request_channel()
942 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;  in sdma_request_channel()
944 	sdma_set_channel_priority(sdmac, MXC_SDMA_DEFAULT_PRIORITY);  in sdma_request_channel()
954 	struct sdma_channel *sdmac = to_sdma_chan(tx->chan);  in sdma_tx_submit()  local
957 	spin_lock_irqsave(&sdmac->lock, flags);  in sdma_tx_submit()
961 	spin_unlock_irqrestore(&sdmac->lock, flags);  in sdma_tx_submit()
968 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_alloc_chan_resources()  local
988 	sdmac->peripheral_type = data->peripheral_type;  in sdma_alloc_chan_resources()
989 	sdmac->event_id0 = data->dma_request;  in sdma_alloc_chan_resources()
991 	clk_enable(sdmac->sdma->clk_ipg);  in sdma_alloc_chan_resources()
992 	clk_enable(sdmac->sdma->clk_ahb);  in sdma_alloc_chan_resources()
994 	ret = sdma_request_channel(sdmac);  in sdma_alloc_chan_resources()
998 	ret = sdma_set_channel_priority(sdmac, prio);  in sdma_alloc_chan_resources()
1002 	dma_async_tx_descriptor_init(&sdmac->desc, chan);  in sdma_alloc_chan_resources()
1003 	sdmac->desc.tx_submit = sdma_tx_submit;  in sdma_alloc_chan_resources()
1005 	sdmac->desc.flags = DMA_CTRL_ACK;  in sdma_alloc_chan_resources()
1012 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_free_chan_resources()  local
1013 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_free_chan_resources()
1017 	if (sdmac->event_id0)  in sdma_free_chan_resources()
1018 		sdma_event_disable(sdmac, sdmac->event_id0);  in sdma_free_chan_resources()
1019 	if (sdmac->event_id1)  in sdma_free_chan_resources()
1020 		sdma_event_disable(sdmac, sdmac->event_id1);  in sdma_free_chan_resources()
1022 	sdmac->event_id0 = 0;  in sdma_free_chan_resources()
1023 	sdmac->event_id1 = 0;  in sdma_free_chan_resources()
1025 	sdma_set_channel_priority(sdmac, 0);  in sdma_free_chan_resources()
1027 	dma_free_coherent(NULL, PAGE_SIZE, sdmac->bd, sdmac->bd_phys);  in sdma_free_chan_resources()
1038 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_slave_sg()  local
1039 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_slave_sg()
1041 	int channel = sdmac->channel;  in sdma_prep_slave_sg()
1044 	if (sdmac->status == DMA_IN_PROGRESS)  in sdma_prep_slave_sg()
1046 	sdmac->status = DMA_IN_PROGRESS;  in sdma_prep_slave_sg()
1048 	sdmac->flags = 0;  in sdma_prep_slave_sg()
1050 	sdmac->buf_tail = 0;  in sdma_prep_slave_sg()
1055 	sdmac->direction = direction;  in sdma_prep_slave_sg()
1056 	ret = sdma_load_context(sdmac);  in sdma_prep_slave_sg()
1067 	sdmac->chn_count = 0;  in sdma_prep_slave_sg()
1069 		struct sdma_buffer_descriptor *bd = &sdmac->bd[i];  in sdma_prep_slave_sg()
1084 		sdmac->chn_count += count;  in sdma_prep_slave_sg()
1086 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) {  in sdma_prep_slave_sg()
1091 		switch (sdmac->word_size) {  in sdma_prep_slave_sg()
1125 	sdmac->num_bd = sg_len;  in sdma_prep_slave_sg()
1126 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;  in sdma_prep_slave_sg()
1128 	return &sdmac->desc;  in sdma_prep_slave_sg()
1130 	sdmac->status = DMA_ERROR;  in sdma_prep_slave_sg()
1139 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_dma_cyclic()  local
1140 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_dma_cyclic()
1142 	int channel = sdmac->channel;  in sdma_prep_dma_cyclic()
1147 	if (sdmac->status == DMA_IN_PROGRESS)  in sdma_prep_dma_cyclic()
1150 	sdmac->status = DMA_IN_PROGRESS;  in sdma_prep_dma_cyclic()
1152 	sdmac->buf_tail = 0;  in sdma_prep_dma_cyclic()
1153 	sdmac->period_len = period_len;  in sdma_prep_dma_cyclic()
1155 	sdmac->flags |= IMX_DMA_SG_LOOP;  in sdma_prep_dma_cyclic()
1156 	sdmac->direction = direction;  in sdma_prep_dma_cyclic()
1157 	ret = sdma_load_context(sdmac);  in sdma_prep_dma_cyclic()
1174 		struct sdma_buffer_descriptor *bd = &sdmac->bd[i];  in sdma_prep_dma_cyclic()
1181 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_dma_cyclic()
1183 		if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_dma_cyclic()
1186 			bd->mode.command = sdmac->word_size;  in sdma_prep_dma_cyclic()
1205 	sdmac->num_bd = num_periods;  in sdma_prep_dma_cyclic()
1206 	sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys;  in sdma_prep_dma_cyclic()
1208 	return &sdmac->desc;  in sdma_prep_dma_cyclic()
1210 	sdmac->status = DMA_ERROR;  in sdma_prep_dma_cyclic()
1217 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_config()  local
1220 		sdmac->per_address = dmaengine_cfg->src_addr;  in sdma_config()
1221 		sdmac->watermark_level = dmaengine_cfg->src_maxburst *  in sdma_config()
1223 		sdmac->word_size = dmaengine_cfg->src_addr_width;  in sdma_config()
1225 		sdmac->per_address = dmaengine_cfg->dst_addr;  in sdma_config()
1226 		sdmac->watermark_level = dmaengine_cfg->dst_maxburst *  in sdma_config()
1228 		sdmac->word_size = dmaengine_cfg->dst_addr_width;  in sdma_config()
1230 	sdmac->direction = dmaengine_cfg->direction;  in sdma_config()
1238 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_tx_status()  local
1241 	if (sdmac->flags & IMX_DMA_SG_LOOP)  in sdma_tx_status()
1242 		residue = (sdmac->num_bd - sdmac->buf_tail) * sdmac->period_len;  in sdma_tx_status()
1244 		residue = sdmac->chn_count - sdmac->chn_real_count;  in sdma_tx_status()
1249 	return sdmac->status;  in sdma_tx_status()
1254 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_issue_pending()  local
1255 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_issue_pending()
1257 	if (sdmac->status == DMA_IN_PROGRESS)  in sdma_issue_pending()
1258 		sdma_enable_channel(sdma, sdmac->channel);  in sdma_issue_pending()
1422 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_filter_fn()  local
1428 	sdmac->data = *data;  in sdma_filter_fn()
1429 	chan->private = &sdmac->data;  in sdma_filter_fn()
1529 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_probe()  local
1531 		sdmac->sdma = sdma;  in sdma_probe()
1532 		spin_lock_init(&sdmac->lock);  in sdma_probe()
1534 		sdmac->chan.device = &sdma->dma_device;  in sdma_probe()
1535 		dma_cookie_init(&sdmac->chan);  in sdma_probe()
1536 		sdmac->channel = i;  in sdma_probe()
1538 		tasklet_init(&sdmac->tasklet, sdma_tasklet,  in sdma_probe()
1539 			     (unsigned long) sdmac);  in sdma_probe()
1546 			list_add_tail(&sdmac->chan.device_node,  in sdma_probe()
1633 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_remove()  local
1635 		tasklet_kill(&sdmac->tasklet);  in sdma_remove()