187 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_terminate_all()  local
192 	dev_dbg(chan2dev(chan), "%s: ch=%p\n", __func__, ch);  in moxart_terminate_all()
194 	spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_terminate_all()
196 	if (ch->desc) {  in moxart_terminate_all()
197 		moxart_dma_desc_free(&ch->desc->vd);  in moxart_terminate_all()
198 		ch->desc = NULL;  in moxart_terminate_all()
201 	ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_terminate_all()
203 	writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_terminate_all()
205 	vchan_get_all_descriptors(&ch->vc, &head);  in moxart_terminate_all()
206 	spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_terminate_all()
207 	vchan_dma_desc_free_list(&ch->vc, &head);  in moxart_terminate_all()
215 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_slave_config()  local
218 	ch->cfg = *cfg;  in moxart_slave_config()
220 	ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_slave_config()
225 	switch (ch->cfg.src_addr_width) {  in moxart_slave_config()
228 		if (ch->cfg.direction != DMA_MEM_TO_DEV)  in moxart_slave_config()
235 		if (ch->cfg.direction != DMA_MEM_TO_DEV)  in moxart_slave_config()
242 		if (ch->cfg.direction != DMA_MEM_TO_DEV)  in moxart_slave_config()
251 	if (ch->cfg.direction == DMA_MEM_TO_DEV) {  in moxart_slave_config()
254 		ctrl |= (ch->line_reqno << 16 &  in moxart_slave_config()
259 		ctrl |= (ch->line_reqno << 24 &  in moxart_slave_config()
263 	writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_slave_config()
273 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_prep_slave_sg()  local
288 		dev_addr = ch->cfg.src_addr;  in moxart_prep_slave_sg()
289 		dev_width = ch->cfg.src_addr_width;  in moxart_prep_slave_sg()
291 		dev_addr = ch->cfg.dst_addr;  in moxart_prep_slave_sg()
292 		dev_width = ch->cfg.dst_addr_width;  in moxart_prep_slave_sg()
326 	ch->error = 0;  in moxart_prep_slave_sg()
328 	return vchan_tx_prep(&ch->vc, &d->vd, tx_flags);  in moxart_prep_slave_sg()
336 	struct moxart_chan *ch;  in moxart_of_xlate()  local
342 	ch = to_moxart_dma_chan(chan);  in moxart_of_xlate()
343 	ch->line_reqno = dma_spec->args[0];  in moxart_of_xlate()
350 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_alloc_chan_resources()  local
353 		__func__, ch->ch_num);  in moxart_alloc_chan_resources()
354 	ch->allocated = 1;  in moxart_alloc_chan_resources()
361 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_free_chan_resources()  local
363 	vchan_free_chan_resources(&ch->vc);  in moxart_free_chan_resources()
366 		__func__, ch->ch_num);  in moxart_free_chan_resources()
367 	ch->allocated = 0;  in moxart_free_chan_resources()
370 static void moxart_dma_set_params(struct moxart_chan *ch, dma_addr_t src_addr,  in moxart_dma_set_params()  argument
373 	writel(src_addr, ch->base + REG_OFF_ADDRESS_SOURCE);  in moxart_dma_set_params()
374 	writel(dst_addr, ch->base + REG_OFF_ADDRESS_DEST);  in moxart_dma_set_params()
377 static void moxart_set_transfer_params(struct moxart_chan *ch, unsigned int len)  in moxart_set_transfer_params()  argument
379 	struct moxart_desc *d = ch->desc;  in moxart_set_transfer_params()
388 	writel(d->dma_cycles, ch->base + REG_OFF_CYCLES);  in moxart_set_transfer_params()
390 	dev_dbg(chan2dev(&ch->vc.chan), "%s: set %u DMA cycles (len=%u)\n",  in moxart_set_transfer_params()
394 static void moxart_start_dma(struct moxart_chan *ch)  in moxart_start_dma()  argument
398 	ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_start_dma()
400 	writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_start_dma()
403 static void moxart_dma_start_sg(struct moxart_chan *ch, unsigned int idx)  in moxart_dma_start_sg()  argument
405 	struct moxart_desc *d = ch->desc;  in moxart_dma_start_sg()
406 	struct moxart_sg *sg = ch->desc->sg + idx;  in moxart_dma_start_sg()
408 	if (ch->desc->dma_dir == DMA_MEM_TO_DEV)  in moxart_dma_start_sg()
409 		moxart_dma_set_params(ch, sg->addr, d->dev_addr);  in moxart_dma_start_sg()
410 	else if (ch->desc->dma_dir == DMA_DEV_TO_MEM)  in moxart_dma_start_sg()
411 		moxart_dma_set_params(ch, d->dev_addr, sg->addr);  in moxart_dma_start_sg()
413 	moxart_set_transfer_params(ch, sg->len);  in moxart_dma_start_sg()
415 	moxart_start_dma(ch);  in moxart_dma_start_sg()
420 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_dma_start_desc()  local
423 	vd = vchan_next_desc(&ch->vc);  in moxart_dma_start_desc()
426 		ch->desc = NULL;  in moxart_dma_start_desc()
432 	ch->desc = to_moxart_dma_desc(&vd->tx);  in moxart_dma_start_desc()
433 	ch->sgidx = 0;  in moxart_dma_start_desc()
435 	moxart_dma_start_sg(ch, 0);  in moxart_dma_start_desc()
440 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_issue_pending()  local
443 	spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_issue_pending()
444 	if (vchan_issue_pending(&ch->vc) && !ch->desc)  in moxart_issue_pending()
446 	spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_issue_pending()
461 static size_t moxart_dma_desc_size_in_flight(struct moxart_chan *ch)  in moxart_dma_desc_size_in_flight()  argument
466 	size = moxart_dma_desc_size(ch->desc, ch->sgidx);  in moxart_dma_desc_size_in_flight()
467 	cycles = readl(ch->base + REG_OFF_CYCLES);  in moxart_dma_desc_size_in_flight()
468 	completed_cycles = (ch->desc->dma_cycles - cycles);  in moxart_dma_desc_size_in_flight()
469 	size -= completed_cycles << es_bytes[ch->desc->es];  in moxart_dma_desc_size_in_flight()
471 	dev_dbg(chan2dev(&ch->vc.chan), "%s: size=%zu\n", __func__, size);  in moxart_dma_desc_size_in_flight()
480 	struct moxart_chan *ch = to_moxart_dma_chan(chan);  in moxart_tx_status()  local
491 	spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_tx_status()
492 	vd = vchan_find_desc(&ch->vc, cookie);  in moxart_tx_status()
496 	} else if (ch->desc && ch->desc->vd.tx.cookie == cookie) {  in moxart_tx_status()
497 		txstate->residue = moxart_dma_desc_size_in_flight(ch);  in moxart_tx_status()
499 	spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_tx_status()
501 	if (ch->error)  in moxart_tx_status()
524 	struct moxart_chan *ch = &mc->slave_chans[0];  in moxart_dma_interrupt()  local
529 	dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__);  in moxart_dma_interrupt()
531 	for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) {  in moxart_dma_interrupt()
532 		if (!ch->allocated)  in moxart_dma_interrupt()
535 		ctrl = readl(ch->base + REG_OFF_CTRL);  in moxart_dma_interrupt()
537 		dev_dbg(chan2dev(&ch->vc.chan), "%s: ch=%p ch->base=%p ctrl=%x\n",  in moxart_dma_interrupt()
538 			__func__, ch, ch->base, ctrl);  in moxart_dma_interrupt()
542 			if (ch->desc) {  in moxart_dma_interrupt()
543 				spin_lock_irqsave(&ch->vc.lock, flags);  in moxart_dma_interrupt()
544 				if (++ch->sgidx < ch->desc->sglen) {  in moxart_dma_interrupt()
545 					moxart_dma_start_sg(ch, ch->sgidx);  in moxart_dma_interrupt()
547 					vchan_cookie_complete(&ch->desc->vd);  in moxart_dma_interrupt()
548 					moxart_dma_start_desc(&ch->vc.chan);  in moxart_dma_interrupt()
550 				spin_unlock_irqrestore(&ch->vc.lock, flags);  in moxart_dma_interrupt()
556 			ch->error = 1;  in moxart_dma_interrupt()
559 		writel(ctrl, ch->base + REG_OFF_CTRL);  in moxart_dma_interrupt()
573 	struct moxart_chan *ch;  in moxart_probe()  local
599 	ch = &mdc->slave_chans[0];  in moxart_probe()
600 	for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) {  in moxart_probe()
601 		ch->ch_num = i;  in moxart_probe()
602 		ch->base = dma_base_addr + i * REG_OFF_CHAN_SIZE;  in moxart_probe()
603 		ch->allocated = 0;  in moxart_probe()
605 		ch->vc.desc_free = moxart_dma_desc_free;  in moxart_probe()
606 		vchan_init(&ch->vc, &mdc->dma_slave);  in moxart_probe()
609 			__func__, i, ch->ch_num, ch->base);  in moxart_probe()