170 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)  in to_omap_dma_chan()  argument
172 	return container_of(c, struct omap_chan, vc.chan);  in to_omap_dma_chan()
244 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)  in omap_dma_chan_write()  argument
246 	const struct omap_dma_reg *r = c->reg_map + reg;  in omap_dma_chan_write()
248 	omap_dma_write(val, r->type, c->channel_base + r->offset);  in omap_dma_chan_write()
251 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)  in omap_dma_chan_read()  argument
253 	const struct omap_dma_reg *r = c->reg_map + reg;  in omap_dma_chan_read()
255 	return omap_dma_read(r->type, c->channel_base + r->offset);  in omap_dma_chan_read()
258 static void omap_dma_clear_csr(struct omap_chan *c)  in omap_dma_clear_csr()  argument
261 		omap_dma_chan_read(c, CSR);  in omap_dma_clear_csr()
263 		omap_dma_chan_write(c, CSR, ~0);  in omap_dma_clear_csr()
266 static unsigned omap_dma_get_csr(struct omap_chan *c)  in omap_dma_get_csr()  argument
268 	unsigned val = omap_dma_chan_read(c, CSR);  in omap_dma_get_csr()
271 		omap_dma_chan_write(c, CSR, val);  in omap_dma_get_csr()
276 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,  in omap_dma_assign()  argument
279 	c->channel_base = od->base + od->plat->channel_stride * lch;  in omap_dma_assign()
281 	od->lch_map[lch] = c;  in omap_dma_assign()
284 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)  in omap_dma_start()  argument
286 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_start()
289 		omap_dma_chan_write(c, CPC, 0);  in omap_dma_start()
291 		omap_dma_chan_write(c, CDAC, 0);  in omap_dma_start()
293 	omap_dma_clear_csr(c);  in omap_dma_start()
296 	omap_dma_chan_write(c, CICR, d->cicr);  in omap_dma_start()
299 	omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);  in omap_dma_start()
302 static void omap_dma_stop(struct omap_chan *c)  in omap_dma_stop()  argument
304 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_stop()
308 	omap_dma_chan_write(c, CICR, 0);  in omap_dma_stop()
310 	omap_dma_clear_csr(c);  in omap_dma_stop()
312 	val = omap_dma_chan_read(c, CCR);  in omap_dma_stop()
322 		val = omap_dma_chan_read(c, CCR);  in omap_dma_stop()
324 		omap_dma_chan_write(c, CCR, val);  in omap_dma_stop()
328 			val = omap_dma_chan_read(c, CCR);  in omap_dma_stop()
339 			dev_err(c->vc.chan.device->dev,  in omap_dma_stop()
341 			        c->dma_ch);  in omap_dma_stop()
346 		omap_dma_chan_write(c, CCR, val);  in omap_dma_stop()
351 	if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {  in omap_dma_stop()
352 		val = omap_dma_chan_read(c, CLNK_CTRL);  in omap_dma_stop()
359 		omap_dma_chan_write(c, CLNK_CTRL, val);  in omap_dma_stop()
363 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d,  in omap_dma_start_sg()  argument
379 	omap_dma_chan_write(c, cxsa, sg->addr);  in omap_dma_start_sg()
380 	omap_dma_chan_write(c, cxei, 0);  in omap_dma_start_sg()
381 	omap_dma_chan_write(c, cxfi, 0);  in omap_dma_start_sg()
382 	omap_dma_chan_write(c, CEN, sg->en);  in omap_dma_start_sg()
383 	omap_dma_chan_write(c, CFN, sg->fn);  in omap_dma_start_sg()
385 	omap_dma_start(c, d);  in omap_dma_start_sg()
388 static void omap_dma_start_desc(struct omap_chan *c)  in omap_dma_start_desc()  argument
390 	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);  in omap_dma_start_desc()
395 		c->desc = NULL;  in omap_dma_start_desc()
401 	c->desc = d = to_omap_dma_desc(&vd->tx);  in omap_dma_start_desc()
402 	c->sgidx = 0;  in omap_dma_start_desc()
411 	omap_dma_chan_write(c, CCR, d->ccr);  in omap_dma_start_desc()
413 		omap_dma_chan_write(c, CCR2, d->ccr >> 16);  in omap_dma_start_desc()
425 	omap_dma_chan_write(c, cxsa, d->dev_addr);  in omap_dma_start_desc()
426 	omap_dma_chan_write(c, cxei, 0);  in omap_dma_start_desc()
427 	omap_dma_chan_write(c, cxfi, d->fi);  in omap_dma_start_desc()
428 	omap_dma_chan_write(c, CSDP, d->csdp);  in omap_dma_start_desc()
429 	omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);  in omap_dma_start_desc()
431 	omap_dma_start_sg(c, d, 0);  in omap_dma_start_desc()
436 	struct omap_chan *c = data;  in omap_dma_callback()  local
440 	spin_lock_irqsave(&c->vc.lock, flags);  in omap_dma_callback()
441 	d = c->desc;  in omap_dma_callback()
443 		if (!c->cyclic) {  in omap_dma_callback()
444 			if (++c->sgidx < d->sglen) {  in omap_dma_callback()
445 				omap_dma_start_sg(c, d, c->sgidx);  in omap_dma_callback()
447 				omap_dma_start_desc(c);  in omap_dma_callback()
454 	spin_unlock_irqrestore(&c->vc.lock, flags);  in omap_dma_callback()
474 		struct omap_chan *c = list_first_entry(&head,  in omap_dma_sched()  local
477 		spin_lock_irq(&c->vc.lock);  in omap_dma_sched()
478 		list_del_init(&c->node);  in omap_dma_sched()
479 		omap_dma_start_desc(c);  in omap_dma_sched()
480 		spin_unlock_irq(&c->vc.lock);  in omap_dma_sched()
500 		struct omap_chan *c;  in omap_dma_irq()  local
506 		c = od->lch_map[channel];  in omap_dma_irq()
507 		if (c == NULL) {  in omap_dma_irq()
513 		csr = omap_dma_get_csr(c);  in omap_dma_irq()
516 		omap_dma_callback(channel, csr, c);  in omap_dma_irq()
527 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_alloc_chan_resources()  local
531 		ret = omap_request_dma(c->dma_sig, "DMA engine",  in omap_dma_alloc_chan_resources()
532 				       omap_dma_callback, c, &c->dma_ch);  in omap_dma_alloc_chan_resources()
534 		ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,  in omap_dma_alloc_chan_resources()
535 				       &c->dma_ch);  in omap_dma_alloc_chan_resources()
539 		c->dma_ch, c->dma_sig);  in omap_dma_alloc_chan_resources()
542 		omap_dma_assign(od, c, c->dma_ch);  in omap_dma_alloc_chan_resources()
548 			val = BIT(c->dma_ch);  in omap_dma_alloc_chan_resources()
554 			val &= ~BIT(c->dma_ch);  in omap_dma_alloc_chan_resources()
562 			c->ccr = CCR_OMAP31_DISABLE;  in omap_dma_alloc_chan_resources()
564 			c->ccr |= c->dma_ch + 1;  in omap_dma_alloc_chan_resources()
566 			c->ccr = c->dma_sig & 0x1f;  in omap_dma_alloc_chan_resources()
569 		c->ccr = c->dma_sig & 0x1f;  in omap_dma_alloc_chan_resources()
570 		c->ccr |= (c->dma_sig & ~0x1f) << 14;  in omap_dma_alloc_chan_resources()
573 		c->ccr |= CCR_BUFFERING_DISABLE;  in omap_dma_alloc_chan_resources()
581 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_free_chan_resources()  local
585 		od->irq_enable_mask &= ~BIT(c->dma_ch);  in omap_dma_free_chan_resources()
590 	c->channel_base = NULL;  in omap_dma_free_chan_resources()
591 	od->lch_map[c->dma_ch] = NULL;  in omap_dma_free_chan_resources()
592 	vchan_free_chan_resources(&c->vc);  in omap_dma_free_chan_resources()
593 	omap_free_dma(c->dma_ch);  in omap_dma_free_chan_resources()
595 	dev_dbg(od->ddev.dev, "freeing channel for %u\n", c->dma_sig);  in omap_dma_free_chan_resources()
596 	c->dma_sig = 0;  in omap_dma_free_chan_resources()
636 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)  in omap_dma_chan_read_3_3()  argument
638 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_chan_read_3_3()
641 	val = omap_dma_chan_read(c, reg);  in omap_dma_chan_read_3_3()
643 		val = omap_dma_chan_read(c, reg);  in omap_dma_chan_read_3_3()
648 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)  in omap_dma_get_src_pos()  argument
650 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_get_src_pos()
654 		addr = omap_dma_chan_read(c, CPC);  in omap_dma_get_src_pos()
656 		addr = omap_dma_chan_read_3_3(c, CSAC);  in omap_dma_get_src_pos()
657 		cdac = omap_dma_chan_read_3_3(c, CDAC);  in omap_dma_get_src_pos()
665 			addr = omap_dma_chan_read(c, CSSA);  in omap_dma_get_src_pos()
669 		addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;  in omap_dma_get_src_pos()
674 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)  in omap_dma_get_dst_pos()  argument
676 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_get_dst_pos()
680 		addr = omap_dma_chan_read(c, CPC);  in omap_dma_get_dst_pos()
682 		addr = omap_dma_chan_read_3_3(c, CDAC);  in omap_dma_get_dst_pos()
691 			addr = omap_dma_chan_read(c, CDSA);  in omap_dma_get_dst_pos()
695 		addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;  in omap_dma_get_dst_pos()
703 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_tx_status()  local
712 	spin_lock_irqsave(&c->vc.lock, flags);  in omap_dma_tx_status()
713 	vd = vchan_find_desc(&c->vc, cookie);  in omap_dma_tx_status()
716 	} else if (c->desc && c->desc->vd.tx.cookie == cookie) {  in omap_dma_tx_status()
717 		struct omap_desc *d = c->desc;  in omap_dma_tx_status()
721 			pos = omap_dma_get_src_pos(c);  in omap_dma_tx_status()
723 			pos = omap_dma_get_dst_pos(c);  in omap_dma_tx_status()
731 	spin_unlock_irqrestore(&c->vc.lock, flags);  in omap_dma_tx_status()
738 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_issue_pending()  local
741 	spin_lock_irqsave(&c->vc.lock, flags);  in omap_dma_issue_pending()
742 	if (vchan_issue_pending(&c->vc) && !c->desc) {  in omap_dma_issue_pending()
747 		if (!c->cyclic) {  in omap_dma_issue_pending()
750 			if (list_empty(&c->node))  in omap_dma_issue_pending()
751 				list_add_tail(&c->node, &d->pending);  in omap_dma_issue_pending()
755 			omap_dma_start_desc(c);  in omap_dma_issue_pending()
758 	spin_unlock_irqrestore(&c->vc.lock, flags);  in omap_dma_issue_pending()
766 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_prep_slave_sg()  local
775 		dev_addr = c->cfg.src_addr;  in omap_dma_prep_slave_sg()
776 		dev_width = c->cfg.src_addr_width;  in omap_dma_prep_slave_sg()
777 		burst = c->cfg.src_maxburst;  in omap_dma_prep_slave_sg()
779 		dev_addr = c->cfg.dst_addr;  in omap_dma_prep_slave_sg()
780 		dev_width = c->cfg.dst_addr_width;  in omap_dma_prep_slave_sg()
781 		burst = c->cfg.dst_maxburst;  in omap_dma_prep_slave_sg()
811 	d->ccr = c->ccr | CCR_SYNC_FRAME;  in omap_dma_prep_slave_sg()
834 		d->clnk_ctrl = c->dma_ch;  in omap_dma_prep_slave_sg()
856 	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);  in omap_dma_prep_slave_sg()
864 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_prep_dma_cyclic()  local
872 		dev_addr = c->cfg.src_addr;  in omap_dma_prep_dma_cyclic()
873 		dev_width = c->cfg.src_addr_width;  in omap_dma_prep_dma_cyclic()
874 		burst = c->cfg.src_maxburst;  in omap_dma_prep_dma_cyclic()
876 		dev_addr = c->cfg.dst_addr;  in omap_dma_prep_dma_cyclic()
877 		dev_width = c->cfg.dst_addr_width;  in omap_dma_prep_dma_cyclic()
878 		burst = c->cfg.dst_maxburst;  in omap_dma_prep_dma_cyclic()
913 	d->ccr = c->ccr;  in omap_dma_prep_dma_cyclic()
953 		d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;  in omap_dma_prep_dma_cyclic()
955 	c->cyclic = true;  in omap_dma_prep_dma_cyclic()
957 	return vchan_tx_prep(&c->vc, &d->vd, flags);  in omap_dma_prep_dma_cyclic()
964 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_prep_dma_memcpy()  local
984 	d->ccr = c->ccr;  in omap_dma_prep_dma_memcpy()
1002 	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);  in omap_dma_prep_dma_memcpy()
1007 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_slave_config()  local
1013 	memcpy(&c->cfg, cfg, sizeof(c->cfg));  in omap_dma_slave_config()
1020 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_terminate_all()  local
1021 	struct omap_dmadev *d = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_terminate_all()
1025 	spin_lock_irqsave(&c->vc.lock, flags);  in omap_dma_terminate_all()
1029 	list_del_init(&c->node);  in omap_dma_terminate_all()
1037 	if (c->desc) {  in omap_dma_terminate_all()
1038 		omap_dma_desc_free(&c->desc->vd);  in omap_dma_terminate_all()
1039 		c->desc = NULL;  in omap_dma_terminate_all()
1041 		if (!c->paused)  in omap_dma_terminate_all()
1042 			omap_dma_stop(c);  in omap_dma_terminate_all()
1045 	if (c->cyclic) {  in omap_dma_terminate_all()
1046 		c->cyclic = false;  in omap_dma_terminate_all()
1047 		c->paused = false;  in omap_dma_terminate_all()
1050 	vchan_get_all_descriptors(&c->vc, &head);  in omap_dma_terminate_all()
1051 	spin_unlock_irqrestore(&c->vc.lock, flags);  in omap_dma_terminate_all()
1052 	vchan_dma_desc_free_list(&c->vc, &head);  in omap_dma_terminate_all()
1059 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_pause()  local
1062 	if (!c->cyclic)  in omap_dma_pause()
1065 	if (!c->paused) {  in omap_dma_pause()
1066 		omap_dma_stop(c);  in omap_dma_pause()
1067 		c->paused = true;  in omap_dma_pause()
1075 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_resume()  local
1078 	if (!c->cyclic)  in omap_dma_resume()
1081 	if (c->paused) {  in omap_dma_resume()
1085 		omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);  in omap_dma_resume()
1087 		omap_dma_start(c, c->desc);  in omap_dma_resume()
1088 		c->paused = false;  in omap_dma_resume()
1096 	struct omap_chan *c;  in omap_dma_chan_init()  local
1098 	c = kzalloc(sizeof(*c), GFP_KERNEL);  in omap_dma_chan_init()
1099 	if (!c)  in omap_dma_chan_init()
1102 	c->reg_map = od->reg_map;  in omap_dma_chan_init()
1103 	c->vc.desc_free = omap_dma_desc_free;  in omap_dma_chan_init()
1104 	vchan_init(&c->vc, &od->ddev);  in omap_dma_chan_init()
1105 	INIT_LIST_HEAD(&c->node);  in omap_dma_chan_init()
1114 		struct omap_chan *c = list_first_entry(&od->ddev.channels,  in omap_dma_free()  local
1117 		list_del(&c->vc.chan.device_node);  in omap_dma_free()
1118 		tasklet_kill(&c->vc.task);  in omap_dma_free()
1119 		kfree(c);  in omap_dma_free()
1276 		struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_filter_fn()  local
1280 			c->dma_sig = req;  in omap_dma_filter_fn()