166 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)  in to_omap_dma_chan()  argument
168 	return container_of(c, struct omap_chan, vc.chan);  in to_omap_dma_chan()
240 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)  in omap_dma_chan_write()  argument
242 	const struct omap_dma_reg *r = c->reg_map + reg;  in omap_dma_chan_write()
244 	omap_dma_write(val, r->type, c->channel_base + r->offset);  in omap_dma_chan_write()
247 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)  in omap_dma_chan_read()  argument
249 	const struct omap_dma_reg *r = c->reg_map + reg;  in omap_dma_chan_read()
251 	return omap_dma_read(r->type, c->channel_base + r->offset);  in omap_dma_chan_read()
254 static void omap_dma_clear_csr(struct omap_chan *c)  in omap_dma_clear_csr()  argument
257 		omap_dma_chan_read(c, CSR);  in omap_dma_clear_csr()
259 		omap_dma_chan_write(c, CSR, ~0);  in omap_dma_clear_csr()
262 static unsigned omap_dma_get_csr(struct omap_chan *c)  in omap_dma_get_csr()  argument
264 	unsigned val = omap_dma_chan_read(c, CSR);  in omap_dma_get_csr()
267 		omap_dma_chan_write(c, CSR, val);  in omap_dma_get_csr()
272 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,  in omap_dma_assign()  argument
275 	c->channel_base = od->base + od->plat->channel_stride * lch;  in omap_dma_assign()
277 	od->lch_map[lch] = c;  in omap_dma_assign()
280 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)  in omap_dma_start()  argument
282 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_start()
285 		omap_dma_chan_write(c, CPC, 0);  in omap_dma_start()
287 		omap_dma_chan_write(c, CDAC, 0);  in omap_dma_start()
289 	omap_dma_clear_csr(c);  in omap_dma_start()
292 	omap_dma_chan_write(c, CICR, d->cicr);  in omap_dma_start()
295 	omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);  in omap_dma_start()
298 static void omap_dma_stop(struct omap_chan *c)  in omap_dma_stop()  argument
300 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_stop()
304 	omap_dma_chan_write(c, CICR, 0);  in omap_dma_stop()
306 	omap_dma_clear_csr(c);  in omap_dma_stop()
308 	val = omap_dma_chan_read(c, CCR);  in omap_dma_stop()
318 		val = omap_dma_chan_read(c, CCR);  in omap_dma_stop()
320 		omap_dma_chan_write(c, CCR, val);  in omap_dma_stop()
324 			val = omap_dma_chan_read(c, CCR);  in omap_dma_stop()
335 			dev_err(c->vc.chan.device->dev,  in omap_dma_stop()
337 			        c->dma_ch);  in omap_dma_stop()
342 		omap_dma_chan_write(c, CCR, val);  in omap_dma_stop()
347 	if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {  in omap_dma_stop()
348 		val = omap_dma_chan_read(c, CLNK_CTRL);  in omap_dma_stop()
355 		omap_dma_chan_write(c, CLNK_CTRL, val);  in omap_dma_stop()
359 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d,  in omap_dma_start_sg()  argument
375 	omap_dma_chan_write(c, cxsa, sg->addr);  in omap_dma_start_sg()
376 	omap_dma_chan_write(c, cxei, 0);  in omap_dma_start_sg()
377 	omap_dma_chan_write(c, cxfi, 0);  in omap_dma_start_sg()
378 	omap_dma_chan_write(c, CEN, sg->en);  in omap_dma_start_sg()
379 	omap_dma_chan_write(c, CFN, sg->fn);  in omap_dma_start_sg()
381 	omap_dma_start(c, d);  in omap_dma_start_sg()
384 static void omap_dma_start_desc(struct omap_chan *c)  in omap_dma_start_desc()  argument
386 	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);  in omap_dma_start_desc()
391 		c->desc = NULL;  in omap_dma_start_desc()
397 	c->desc = d = to_omap_dma_desc(&vd->tx);  in omap_dma_start_desc()
398 	c->sgidx = 0;  in omap_dma_start_desc()
407 	omap_dma_chan_write(c, CCR, d->ccr);  in omap_dma_start_desc()
409 		omap_dma_chan_write(c, CCR2, d->ccr >> 16);  in omap_dma_start_desc()
421 	omap_dma_chan_write(c, cxsa, d->dev_addr);  in omap_dma_start_desc()
422 	omap_dma_chan_write(c, cxei, 0);  in omap_dma_start_desc()
423 	omap_dma_chan_write(c, cxfi, d->fi);  in omap_dma_start_desc()
424 	omap_dma_chan_write(c, CSDP, d->csdp);  in omap_dma_start_desc()
425 	omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);  in omap_dma_start_desc()
427 	omap_dma_start_sg(c, d, 0);  in omap_dma_start_desc()
432 	struct omap_chan *c = data;  in omap_dma_callback()  local
436 	spin_lock_irqsave(&c->vc.lock, flags);  in omap_dma_callback()
437 	d = c->desc;  in omap_dma_callback()
439 		if (!c->cyclic) {  in omap_dma_callback()
440 			if (++c->sgidx < d->sglen) {  in omap_dma_callback()
441 				omap_dma_start_sg(c, d, c->sgidx);  in omap_dma_callback()
443 				omap_dma_start_desc(c);  in omap_dma_callback()
450 	spin_unlock_irqrestore(&c->vc.lock, flags);  in omap_dma_callback()
470 		struct omap_chan *c = list_first_entry(&head,  in omap_dma_sched()  local
473 		spin_lock_irq(&c->vc.lock);  in omap_dma_sched()
474 		list_del_init(&c->node);  in omap_dma_sched()
475 		omap_dma_start_desc(c);  in omap_dma_sched()
476 		spin_unlock_irq(&c->vc.lock);  in omap_dma_sched()
496 		struct omap_chan *c;  in omap_dma_irq()  local
502 		c = od->lch_map[channel];  in omap_dma_irq()
503 		if (c == NULL) {  in omap_dma_irq()
509 		csr = omap_dma_get_csr(c);  in omap_dma_irq()
512 		omap_dma_callback(channel, csr, c);  in omap_dma_irq()
523 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_alloc_chan_resources()  local
527 		ret = omap_request_dma(c->dma_sig, "DMA engine",  in omap_dma_alloc_chan_resources()
528 				       omap_dma_callback, c, &c->dma_ch);  in omap_dma_alloc_chan_resources()
530 		ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,  in omap_dma_alloc_chan_resources()
531 				       &c->dma_ch);  in omap_dma_alloc_chan_resources()
535 		c->dma_ch, c->dma_sig);  in omap_dma_alloc_chan_resources()
538 		omap_dma_assign(od, c, c->dma_ch);  in omap_dma_alloc_chan_resources()
544 			val = BIT(c->dma_ch);  in omap_dma_alloc_chan_resources()
550 			val &= ~BIT(c->dma_ch);  in omap_dma_alloc_chan_resources()
558 			c->ccr = CCR_OMAP31_DISABLE;  in omap_dma_alloc_chan_resources()
560 			c->ccr |= c->dma_ch + 1;  in omap_dma_alloc_chan_resources()
562 			c->ccr = c->dma_sig & 0x1f;  in omap_dma_alloc_chan_resources()
565 		c->ccr = c->dma_sig & 0x1f;  in omap_dma_alloc_chan_resources()
566 		c->ccr |= (c->dma_sig & ~0x1f) << 14;  in omap_dma_alloc_chan_resources()
569 		c->ccr |= CCR_BUFFERING_DISABLE;  in omap_dma_alloc_chan_resources()
577 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_free_chan_resources()  local
581 		od->irq_enable_mask &= ~BIT(c->dma_ch);  in omap_dma_free_chan_resources()
586 	c->channel_base = NULL;  in omap_dma_free_chan_resources()
587 	od->lch_map[c->dma_ch] = NULL;  in omap_dma_free_chan_resources()
588 	vchan_free_chan_resources(&c->vc);  in omap_dma_free_chan_resources()
589 	omap_free_dma(c->dma_ch);  in omap_dma_free_chan_resources()
591 	dev_dbg(od->ddev.dev, "freeing channel for %u\n", c->dma_sig);  in omap_dma_free_chan_resources()
631 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)  in omap_dma_chan_read_3_3()  argument
633 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_chan_read_3_3()
636 	val = omap_dma_chan_read(c, reg);  in omap_dma_chan_read_3_3()
638 		val = omap_dma_chan_read(c, reg);  in omap_dma_chan_read_3_3()
643 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)  in omap_dma_get_src_pos()  argument
645 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_get_src_pos()
649 		addr = omap_dma_chan_read(c, CPC);  in omap_dma_get_src_pos()
651 		addr = omap_dma_chan_read_3_3(c, CSAC);  in omap_dma_get_src_pos()
652 		cdac = omap_dma_chan_read_3_3(c, CDAC);  in omap_dma_get_src_pos()
660 			addr = omap_dma_chan_read(c, CSSA);  in omap_dma_get_src_pos()
664 		addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;  in omap_dma_get_src_pos()
669 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)  in omap_dma_get_dst_pos()  argument
671 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_get_dst_pos()
675 		addr = omap_dma_chan_read(c, CPC);  in omap_dma_get_dst_pos()
677 		addr = omap_dma_chan_read_3_3(c, CDAC);  in omap_dma_get_dst_pos()
686 			addr = omap_dma_chan_read(c, CDSA);  in omap_dma_get_dst_pos()
690 		addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;  in omap_dma_get_dst_pos()
698 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_tx_status()  local
707 	spin_lock_irqsave(&c->vc.lock, flags);  in omap_dma_tx_status()
708 	vd = vchan_find_desc(&c->vc, cookie);  in omap_dma_tx_status()
711 	} else if (c->desc && c->desc->vd.tx.cookie == cookie) {  in omap_dma_tx_status()
712 		struct omap_desc *d = c->desc;  in omap_dma_tx_status()
716 			pos = omap_dma_get_src_pos(c);  in omap_dma_tx_status()
718 			pos = omap_dma_get_dst_pos(c);  in omap_dma_tx_status()
726 	spin_unlock_irqrestore(&c->vc.lock, flags);  in omap_dma_tx_status()
733 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_issue_pending()  local
736 	spin_lock_irqsave(&c->vc.lock, flags);  in omap_dma_issue_pending()
737 	if (vchan_issue_pending(&c->vc) && !c->desc) {  in omap_dma_issue_pending()
742 		if (!c->cyclic) {  in omap_dma_issue_pending()
745 			if (list_empty(&c->node))  in omap_dma_issue_pending()
746 				list_add_tail(&c->node, &d->pending);  in omap_dma_issue_pending()
750 			omap_dma_start_desc(c);  in omap_dma_issue_pending()
753 	spin_unlock_irqrestore(&c->vc.lock, flags);  in omap_dma_issue_pending()
761 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_prep_slave_sg()  local
770 		dev_addr = c->cfg.src_addr;  in omap_dma_prep_slave_sg()
771 		dev_width = c->cfg.src_addr_width;  in omap_dma_prep_slave_sg()
772 		burst = c->cfg.src_maxburst;  in omap_dma_prep_slave_sg()
774 		dev_addr = c->cfg.dst_addr;  in omap_dma_prep_slave_sg()
775 		dev_width = c->cfg.dst_addr_width;  in omap_dma_prep_slave_sg()
776 		burst = c->cfg.dst_maxburst;  in omap_dma_prep_slave_sg()
806 	d->ccr = c->ccr | CCR_SYNC_FRAME;  in omap_dma_prep_slave_sg()
829 		d->clnk_ctrl = c->dma_ch;  in omap_dma_prep_slave_sg()
851 	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);  in omap_dma_prep_slave_sg()
859 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_prep_dma_cyclic()  local
867 		dev_addr = c->cfg.src_addr;  in omap_dma_prep_dma_cyclic()
868 		dev_width = c->cfg.src_addr_width;  in omap_dma_prep_dma_cyclic()
869 		burst = c->cfg.src_maxburst;  in omap_dma_prep_dma_cyclic()
871 		dev_addr = c->cfg.dst_addr;  in omap_dma_prep_dma_cyclic()
872 		dev_width = c->cfg.dst_addr_width;  in omap_dma_prep_dma_cyclic()
873 		burst = c->cfg.dst_maxburst;  in omap_dma_prep_dma_cyclic()
908 	d->ccr = c->ccr;  in omap_dma_prep_dma_cyclic()
944 		d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;  in omap_dma_prep_dma_cyclic()
946 	c->cyclic = true;  in omap_dma_prep_dma_cyclic()
948 	return vchan_tx_prep(&c->vc, &d->vd, flags);  in omap_dma_prep_dma_cyclic()
953 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_slave_config()  local
959 	memcpy(&c->cfg, cfg, sizeof(c->cfg));  in omap_dma_slave_config()
966 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_terminate_all()  local
967 	struct omap_dmadev *d = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_terminate_all()
971 	spin_lock_irqsave(&c->vc.lock, flags);  in omap_dma_terminate_all()
975 	list_del_init(&c->node);  in omap_dma_terminate_all()
983 	if (c->desc) {  in omap_dma_terminate_all()
984 		omap_dma_desc_free(&c->desc->vd);  in omap_dma_terminate_all()
985 		c->desc = NULL;  in omap_dma_terminate_all()
987 		if (!c->paused)  in omap_dma_terminate_all()
988 			omap_dma_stop(c);  in omap_dma_terminate_all()
991 	if (c->cyclic) {  in omap_dma_terminate_all()
992 		c->cyclic = false;  in omap_dma_terminate_all()
993 		c->paused = false;  in omap_dma_terminate_all()
996 	vchan_get_all_descriptors(&c->vc, &head);  in omap_dma_terminate_all()
997 	spin_unlock_irqrestore(&c->vc.lock, flags);  in omap_dma_terminate_all()
998 	vchan_dma_desc_free_list(&c->vc, &head);  in omap_dma_terminate_all()
1005 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_pause()  local
1008 	if (!c->cyclic)  in omap_dma_pause()
1011 	if (!c->paused) {  in omap_dma_pause()
1012 		omap_dma_stop(c);  in omap_dma_pause()
1013 		c->paused = true;  in omap_dma_pause()
1021 	struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_resume()  local
1024 	if (!c->cyclic)  in omap_dma_resume()
1027 	if (c->paused) {  in omap_dma_resume()
1031 		omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);  in omap_dma_resume()
1033 		omap_dma_start(c, c->desc);  in omap_dma_resume()
1034 		c->paused = false;  in omap_dma_resume()
1042 	struct omap_chan *c;  in omap_dma_chan_init()  local
1044 	c = kzalloc(sizeof(*c), GFP_KERNEL);  in omap_dma_chan_init()
1045 	if (!c)  in omap_dma_chan_init()
1048 	c->reg_map = od->reg_map;  in omap_dma_chan_init()
1049 	c->dma_sig = dma_sig;  in omap_dma_chan_init()
1050 	c->vc.desc_free = omap_dma_desc_free;  in omap_dma_chan_init()
1051 	vchan_init(&c->vc, &od->ddev);  in omap_dma_chan_init()
1052 	INIT_LIST_HEAD(&c->node);  in omap_dma_chan_init()
1061 		struct omap_chan *c = list_first_entry(&od->ddev.channels,  in omap_dma_free()  local
1064 		list_del(&c->vc.chan.device_node);  in omap_dma_free()
1065 		tasklet_kill(&c->vc.task);  in omap_dma_free()
1066 		kfree(c);  in omap_dma_free()
1211 		struct omap_chan *c = to_omap_dma_chan(chan);  in omap_dma_filter_fn()  local
1214 		return req == c->dma_sig;  in omap_dma_filter_fn()