132 static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c)  in to_bcm2835_dma_chan()  argument
134 	return container_of(c, struct bcm2835_chan, vc.chan);  in to_bcm2835_dma_chan()
188 static void bcm2835_dma_start_desc(struct bcm2835_chan *c)  in bcm2835_dma_start_desc()  argument
190 	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);  in bcm2835_dma_start_desc()
194 		c->desc = NULL;  in bcm2835_dma_start_desc()
200 	c->desc = d = to_bcm2835_dma_desc(&vd->tx);  in bcm2835_dma_start_desc()
202 	writel(d->control_block_base_phys, c->chan_base + BCM2835_DMA_ADDR);  in bcm2835_dma_start_desc()
203 	writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS);  in bcm2835_dma_start_desc()
208 	struct bcm2835_chan *c = data;  in bcm2835_dma_callback()  local
212 	spin_lock_irqsave(&c->vc.lock, flags);  in bcm2835_dma_callback()
215 	writel(BCM2835_DMA_INT, c->chan_base + BCM2835_DMA_CS);  in bcm2835_dma_callback()
217 	d = c->desc;  in bcm2835_dma_callback()
225 	writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS);  in bcm2835_dma_callback()
227 	spin_unlock_irqrestore(&c->vc.lock, flags);  in bcm2835_dma_callback()
234 	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);  in bcm2835_dma_alloc_chan_resources()  local
236 	dev_dbg(c->vc.chan.device->dev,  in bcm2835_dma_alloc_chan_resources()
237 			"Allocating DMA channel %d\n", c->ch);  in bcm2835_dma_alloc_chan_resources()
239 	return request_irq(c->irq_number,  in bcm2835_dma_alloc_chan_resources()
240 			bcm2835_dma_callback, 0, "DMA IRQ", c);  in bcm2835_dma_alloc_chan_resources()
245 	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);  in bcm2835_dma_free_chan_resources()  local
247 	vchan_free_chan_resources(&c->vc);  in bcm2835_dma_free_chan_resources()
248 	free_irq(c->irq_number, c);  in bcm2835_dma_free_chan_resources()
250 	dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n", c->ch);  in bcm2835_dma_free_chan_resources()
286 	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);  in bcm2835_dma_tx_status()  local
295 	spin_lock_irqsave(&c->vc.lock, flags);  in bcm2835_dma_tx_status()
296 	vd = vchan_find_desc(&c->vc, cookie);  in bcm2835_dma_tx_status()
300 	} else if (c->desc && c->desc->vd.tx.cookie == cookie) {  in bcm2835_dma_tx_status()
301 		struct bcm2835_desc *d = c->desc;  in bcm2835_dma_tx_status()
305 			pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD);  in bcm2835_dma_tx_status()
307 			pos = readl(c->chan_base + BCM2835_DMA_DEST_AD);  in bcm2835_dma_tx_status()
316 	spin_unlock_irqrestore(&c->vc.lock, flags);  in bcm2835_dma_tx_status()
323 	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);  in bcm2835_dma_issue_pending()  local
326 	c->cyclic = true; /* Nothing else is implemented */  in bcm2835_dma_issue_pending()
328 	spin_lock_irqsave(&c->vc.lock, flags);  in bcm2835_dma_issue_pending()
329 	if (vchan_issue_pending(&c->vc) && !c->desc)  in bcm2835_dma_issue_pending()
330 		bcm2835_dma_start_desc(c);  in bcm2835_dma_issue_pending()
332 	spin_unlock_irqrestore(&c->vc.lock, flags);  in bcm2835_dma_issue_pending()
340 	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);  in bcm2835_dma_prep_dma_cyclic()  local
354 		dev_addr = c->cfg.src_addr;  in bcm2835_dma_prep_dma_cyclic()
355 		dev_width = c->cfg.src_addr_width;  in bcm2835_dma_prep_dma_cyclic()
358 		dev_addr = c->cfg.dst_addr;  in bcm2835_dma_prep_dma_cyclic()
359 		dev_width = c->cfg.dst_addr_width;  in bcm2835_dma_prep_dma_cyclic()
418 		if (c->dreq != 0)  in bcm2835_dma_prep_dma_cyclic()
420 				BCM2835_DMA_PER_MAP(c->dreq);  in bcm2835_dma_prep_dma_cyclic()
436 	return vchan_tx_prep(&c->vc, &d->vd, flags);  in bcm2835_dma_prep_dma_cyclic()
442 	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);  in bcm2835_dma_slave_config()  local
452 	c->cfg = *cfg;  in bcm2835_dma_slave_config()
459 	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);  in bcm2835_dma_terminate_all()  local
460 	struct bcm2835_dmadev *d = to_bcm2835_dma_dev(c->vc.chan.device);  in bcm2835_dma_terminate_all()
465 	spin_lock_irqsave(&c->vc.lock, flags);  in bcm2835_dma_terminate_all()
469 	list_del_init(&c->node);  in bcm2835_dma_terminate_all()
477 	if (c->desc) {  in bcm2835_dma_terminate_all()
478 		bcm2835_dma_desc_free(&c->desc->vd);  in bcm2835_dma_terminate_all()
479 		c->desc = NULL;  in bcm2835_dma_terminate_all()
480 		bcm2835_dma_abort(c->chan_base);  in bcm2835_dma_terminate_all()
484 			if (!(readl(c->chan_base + BCM2835_DMA_CS) &  in bcm2835_dma_terminate_all()
495 	vchan_get_all_descriptors(&c->vc, &head);  in bcm2835_dma_terminate_all()
496 	spin_unlock_irqrestore(&c->vc.lock, flags);  in bcm2835_dma_terminate_all()
497 	vchan_dma_desc_free_list(&c->vc, &head);  in bcm2835_dma_terminate_all()
504 	struct bcm2835_chan *c;  in bcm2835_dma_chan_init()  local
506 	c = devm_kzalloc(d->ddev.dev, sizeof(*c), GFP_KERNEL);  in bcm2835_dma_chan_init()
507 	if (!c)  in bcm2835_dma_chan_init()
510 	c->vc.desc_free = bcm2835_dma_desc_free;  in bcm2835_dma_chan_init()
511 	vchan_init(&c->vc, &d->ddev);  in bcm2835_dma_chan_init()
512 	INIT_LIST_HEAD(&c->node);  in bcm2835_dma_chan_init()
514 	c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id);  in bcm2835_dma_chan_init()
515 	c->ch = chan_id;  in bcm2835_dma_chan_init()
516 	c->irq_number = irq;  in bcm2835_dma_chan_init()
523 	struct bcm2835_chan *c, *next;  in bcm2835_dma_free()  local
525 	list_for_each_entry_safe(c, next, &od->ddev.channels,  in bcm2835_dma_free()
527 		list_del(&c->vc.chan.device_node);  in bcm2835_dma_free()
528 		tasklet_kill(&c->vc.task);  in bcm2835_dma_free()