165 	struct jz4780_dma_chan *jzchan)  in jz4780_dma_chan_parent()  argument
167 	return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev,  in jz4780_dma_chan_parent()
184 	struct jz4780_dma_chan *jzchan, unsigned int count,  in jz4780_dma_desc_alloc()  argument
196 	desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT,  in jz4780_dma_desc_alloc()
211 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan);  in jz4780_dma_desc_free()  local
213 	dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys);  in jz4780_dma_desc_free()
241 static uint32_t jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan,  in jz4780_dma_setup_hwdesc()  argument
245 	struct dma_slave_config *config = &jzchan->config;  in jz4780_dma_setup_hwdesc()
253 		desc->drt = jzchan->transfer_type;  in jz4780_dma_setup_hwdesc()
261 		desc->drt = jzchan->transfer_type;  in jz4780_dma_setup_hwdesc()
275 	jzchan->transfer_shift = ord;  in jz4780_dma_setup_hwdesc()
299 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);  in jz4780_dma_prep_slave_sg()  local
304 	desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE);  in jz4780_dma_prep_slave_sg()
309 		err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i],  in jz4780_dma_prep_slave_sg()
333 	return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);  in jz4780_dma_prep_slave_sg()
341 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);  in jz4780_dma_prep_dma_cyclic()  local
351 	desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC);  in jz4780_dma_prep_dma_cyclic()
356 		err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr,  in jz4780_dma_prep_dma_cyclic()
383 	return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);  in jz4780_dma_prep_dma_cyclic()
390 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);  in jz4780_dma_prep_dma_memcpy()  local
395 	desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY);  in jz4780_dma_prep_dma_memcpy()
412 	return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);  in jz4780_dma_prep_dma_memcpy()
415 static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan)  in jz4780_dma_begin()  argument
417 	struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);  in jz4780_dma_begin()
422 	if (!jzchan->desc) {  in jz4780_dma_begin()
423 		vdesc = vchan_next_desc(&jzchan->vchan);  in jz4780_dma_begin()
429 		jzchan->desc = to_jz4780_dma_desc(vdesc);  in jz4780_dma_begin()
430 		jzchan->curr_hwdesc = 0;  in jz4780_dma_begin()
432 		if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) {  in jz4780_dma_begin()
447 			for (i = 0; i < jzchan->desc->count; i++)  in jz4780_dma_begin()
448 				jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK;  in jz4780_dma_begin()
456 		jzchan->curr_hwdesc =  in jz4780_dma_begin()
457 			(jzchan->curr_hwdesc + 1) % jzchan->desc->count;  in jz4780_dma_begin()
461 	jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), JZ_DMA_DCS_DES8);  in jz4780_dma_begin()
464 	desc_phys = jzchan->desc->desc_phys +  in jz4780_dma_begin()
465 		    (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc));  in jz4780_dma_begin()
466 	jz4780_dma_writel(jzdma, JZ_DMA_REG_DDA(jzchan->id), desc_phys);  in jz4780_dma_begin()
467 	jz4780_dma_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id));  in jz4780_dma_begin()
470 	jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id),  in jz4780_dma_begin()
476 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);  in jz4780_dma_issue_pending()  local
479 	spin_lock_irqsave(&jzchan->vchan.lock, flags);  in jz4780_dma_issue_pending()
481 	if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc)  in jz4780_dma_issue_pending()
482 		jz4780_dma_begin(jzchan);  in jz4780_dma_issue_pending()
484 	spin_unlock_irqrestore(&jzchan->vchan.lock, flags);  in jz4780_dma_issue_pending()
487 static int jz4780_dma_terminate_all(struct jz4780_dma_chan *jzchan)  in jz4780_dma_terminate_all()  argument
489 	struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);  in jz4780_dma_terminate_all()
493 	spin_lock_irqsave(&jzchan->vchan.lock, flags);  in jz4780_dma_terminate_all()
496 	jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0);  in jz4780_dma_terminate_all()
497 	if (jzchan->desc) {  in jz4780_dma_terminate_all()
498 		jz4780_dma_desc_free(&jzchan->desc->vdesc);  in jz4780_dma_terminate_all()
499 		jzchan->desc = NULL;  in jz4780_dma_terminate_all()
502 	vchan_get_all_descriptors(&jzchan->vchan, &head);  in jz4780_dma_terminate_all()
504 	spin_unlock_irqrestore(&jzchan->vchan.lock, flags);  in jz4780_dma_terminate_all()
506 	vchan_dma_desc_free_list(&jzchan->vchan, &head);  in jz4780_dma_terminate_all()
510 static int jz4780_dma_slave_config(struct jz4780_dma_chan *jzchan,  in jz4780_dma_slave_config()  argument
518 	memcpy(&jzchan->config, config, sizeof(jzchan->config));  in jz4780_dma_slave_config()
523 static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan,  in jz4780_dma_desc_residue()  argument
526 	struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);  in jz4780_dma_desc_residue()
533 		residue += desc->desc[i].dtc << jzchan->transfer_shift;  in jz4780_dma_desc_residue()
537 					 JZ_DMA_REG_DTC(jzchan->id));  in jz4780_dma_desc_residue()
538 		residue += count << jzchan->transfer_shift;  in jz4780_dma_desc_residue()
547 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);  in jz4780_dma_tx_status()  local
556 	spin_lock_irqsave(&jzchan->vchan.lock, flags);  in jz4780_dma_tx_status()
558 	vdesc = vchan_find_desc(&jzchan->vchan, cookie);  in jz4780_dma_tx_status()
561 		txstate->residue = jz4780_dma_desc_residue(jzchan,  in jz4780_dma_tx_status()
563 	} else if (cookie == jzchan->desc->vdesc.tx.cookie) {  in jz4780_dma_tx_status()
564 		txstate->residue = jz4780_dma_desc_residue(jzchan, jzchan->desc,  in jz4780_dma_tx_status()
565 			  (jzchan->curr_hwdesc + 1) % jzchan->desc->count);  in jz4780_dma_tx_status()
569 	if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc  in jz4780_dma_tx_status()
570 		&& jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT))  in jz4780_dma_tx_status()
573 	spin_unlock_irqrestore(&jzchan->vchan.lock, flags);  in jz4780_dma_tx_status()
578 	struct jz4780_dma_chan *jzchan)  in jz4780_dma_chan_irq()  argument
582 	spin_lock(&jzchan->vchan.lock);  in jz4780_dma_chan_irq()
584 	dcs = jz4780_dma_readl(jzdma, JZ_DMA_REG_DCS(jzchan->id));  in jz4780_dma_chan_irq()
585 	jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0);  in jz4780_dma_chan_irq()
588 		dev_warn(&jzchan->vchan.chan.dev->device,  in jz4780_dma_chan_irq()
593 		dev_warn(&jzchan->vchan.chan.dev->device,  in jz4780_dma_chan_irq()
597 	if (jzchan->desc) {  in jz4780_dma_chan_irq()
598 		jzchan->desc->status = dcs;  in jz4780_dma_chan_irq()
601 			if (jzchan->desc->type == DMA_CYCLIC) {  in jz4780_dma_chan_irq()
602 				vchan_cyclic_callback(&jzchan->desc->vdesc);  in jz4780_dma_chan_irq()
604 				vchan_cookie_complete(&jzchan->desc->vdesc);  in jz4780_dma_chan_irq()
605 				jzchan->desc = NULL;  in jz4780_dma_chan_irq()
608 			jz4780_dma_begin(jzchan);  in jz4780_dma_chan_irq()
611 		dev_err(&jzchan->vchan.chan.dev->device,  in jz4780_dma_chan_irq()
615 	spin_unlock(&jzchan->vchan.lock);  in jz4780_dma_chan_irq()
646 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);  in jz4780_dma_alloc_chan_resources()  local
648 	jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device),  in jz4780_dma_alloc_chan_resources()
652 	if (!jzchan->desc_pool) {  in jz4780_dma_alloc_chan_resources()
663 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);  in jz4780_dma_free_chan_resources()  local
665 	vchan_free_chan_resources(&jzchan->vchan);  in jz4780_dma_free_chan_resources()
666 	dma_pool_destroy(jzchan->desc_pool);  in jz4780_dma_free_chan_resources()
667 	jzchan->desc_pool = NULL;  in jz4780_dma_free_chan_resources()
672 	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);  in jz4780_dma_filter_fn()  local
673 	struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);  in jz4780_dma_filter_fn()
677 		if (data->channel != jzchan->id)  in jz4780_dma_filter_fn()
679 	} else if (jzdma->chan_reserved & BIT(jzchan->id)) {  in jz4780_dma_filter_fn()
683 	jzchan->transfer_type = data->transfer_type;  in jz4780_dma_filter_fn()
725 	struct jz4780_dma_chan *jzchan;  in jz4780_dma_probe()  local
806 		jzchan = &jzdma->chan[i];  in jz4780_dma_probe()
807 		jzchan->id = i;  in jz4780_dma_probe()
809 		vchan_init(&jzchan->vchan, dd);  in jz4780_dma_probe()
810 		jzchan->vchan.desc_free = jz4780_dma_desc_free;  in jz4780_dma_probe()