181 typedef void (*dma_isr_handler)(struct tegra_dma_channel *tdc,
242 static inline void tdc_write(struct tegra_dma_channel *tdc,  in tdc_write()  argument
245 	writel(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg);  in tdc_write()
248 static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)  in tdc_read()  argument
250 	return readl(tdc->tdma->base_addr + tdc->chan_base_offset + reg);  in tdc_read()
264 static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)  in tdc2dev()  argument
266 	return &tdc->dma_chan.dev->device;  in tdc2dev()
275 		struct tegra_dma_channel *tdc)  in tegra_dma_desc_get()  argument
280 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_desc_get()
283 	list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) {  in tegra_dma_desc_get()
286 			spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_desc_get()
292 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_desc_get()
297 		dev_err(tdc2dev(tdc), "dma_desc alloc failed\n");  in tegra_dma_desc_get()
301 	dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan);  in tegra_dma_desc_get()
307 static void tegra_dma_desc_put(struct tegra_dma_channel *tdc,  in tegra_dma_desc_put()  argument
312 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_desc_put()
314 		list_splice_init(&dma_desc->tx_list, &tdc->free_sg_req);  in tegra_dma_desc_put()
315 	list_add_tail(&dma_desc->node, &tdc->free_dma_desc);  in tegra_dma_desc_put()
316 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_desc_put()
320 		struct tegra_dma_channel *tdc)  in tegra_dma_sg_req_get()  argument
325 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_sg_req_get()
326 	if (!list_empty(&tdc->free_sg_req)) {  in tegra_dma_sg_req_get()
327 		sg_req = list_first_entry(&tdc->free_sg_req,  in tegra_dma_sg_req_get()
330 		spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_sg_req_get()
333 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_sg_req_get()
337 		dev_err(tdc2dev(tdc), "sg_req alloc failed\n");  in tegra_dma_sg_req_get()
344 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);  in tegra_dma_slave_config()  local
346 	if (!list_empty(&tdc->pending_sg_req)) {  in tegra_dma_slave_config()
347 		dev_err(tdc2dev(tdc), "Configuration not allowed\n");  in tegra_dma_slave_config()
351 	memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));  in tegra_dma_slave_config()
352 	if (!tdc->slave_id)  in tegra_dma_slave_config()
353 		tdc->slave_id = sconfig->slave_id;  in tegra_dma_slave_config()
354 	tdc->config_init = true;  in tegra_dma_slave_config()
358 static void tegra_dma_global_pause(struct tegra_dma_channel *tdc,  in tegra_dma_global_pause()  argument
361 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_global_pause()
369 static void tegra_dma_global_resume(struct tegra_dma_channel *tdc)  in tegra_dma_global_resume()  argument
371 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_global_resume()
377 static void tegra_dma_pause(struct tegra_dma_channel *tdc,  in tegra_dma_pause()  argument
380 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_pause()
383 		tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE,  in tegra_dma_pause()
388 		tegra_dma_global_pause(tdc, wait_for_burst_complete);  in tegra_dma_pause()
392 static void tegra_dma_resume(struct tegra_dma_channel *tdc)  in tegra_dma_resume()  argument
394 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_resume()
397 		tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0);  in tegra_dma_resume()
399 		tegra_dma_global_resume(tdc);  in tegra_dma_resume()
403 static void tegra_dma_stop(struct tegra_dma_channel *tdc)  in tegra_dma_stop()  argument
409 	csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);  in tegra_dma_stop()
411 	tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr);  in tegra_dma_stop()
415 	tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr);  in tegra_dma_stop()
418 	status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);  in tegra_dma_stop()
420 		dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);  in tegra_dma_stop()
421 		tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status);  in tegra_dma_stop()
423 	tdc->busy = false;  in tegra_dma_stop()
426 static void tegra_dma_start(struct tegra_dma_channel *tdc,  in tegra_dma_start()  argument
431 	tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, ch_regs->csr);  in tegra_dma_start()
432 	tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_regs->apb_seq);  in tegra_dma_start()
433 	tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_regs->apb_ptr);  in tegra_dma_start()
434 	tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_regs->ahb_seq);  in tegra_dma_start()
435 	tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_regs->ahb_ptr);  in tegra_dma_start()
436 	if (tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_start()
437 		tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, ch_regs->wcount);  in tegra_dma_start()
440 	tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,  in tegra_dma_start()
444 static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc,  in tegra_dma_configure_for_next()  argument
460 	tegra_dma_pause(tdc, false);  in tegra_dma_configure_for_next()
461 	status  = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);  in tegra_dma_configure_for_next()
468 		dev_err(tdc2dev(tdc),  in tegra_dma_configure_for_next()
470 		tegra_dma_resume(tdc);  in tegra_dma_configure_for_next()
475 	tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, nsg_req->ch_regs.apb_ptr);  in tegra_dma_configure_for_next()
476 	tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr);  in tegra_dma_configure_for_next()
477 	if (tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_configure_for_next()
478 		tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT,  in tegra_dma_configure_for_next()
480 	tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,  in tegra_dma_configure_for_next()
484 	tegra_dma_resume(tdc);  in tegra_dma_configure_for_next()
487 static void tdc_start_head_req(struct tegra_dma_channel *tdc)  in tdc_start_head_req()  argument
491 	if (list_empty(&tdc->pending_sg_req))  in tdc_start_head_req()
494 	sg_req = list_first_entry(&tdc->pending_sg_req,  in tdc_start_head_req()
496 	tegra_dma_start(tdc, sg_req);  in tdc_start_head_req()
498 	tdc->busy = true;  in tdc_start_head_req()
501 static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc)  in tdc_configure_next_head_desc()  argument
506 	if (list_empty(&tdc->pending_sg_req))  in tdc_configure_next_head_desc()
509 	hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);  in tdc_configure_next_head_desc()
510 	if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) {  in tdc_configure_next_head_desc()
513 		tegra_dma_configure_for_next(tdc, hnsgreq);  in tdc_configure_next_head_desc()
517 static inline int get_current_xferred_count(struct tegra_dma_channel *tdc,  in get_current_xferred_count()  argument
523 static void tegra_dma_abort_all(struct tegra_dma_channel *tdc)  in tegra_dma_abort_all()  argument
528 	while (!list_empty(&tdc->pending_sg_req)) {  in tegra_dma_abort_all()
529 		sgreq = list_first_entry(&tdc->pending_sg_req,  in tegra_dma_abort_all()
531 		list_move_tail(&sgreq->node, &tdc->free_sg_req);  in tegra_dma_abort_all()
535 			list_add_tail(&dma_desc->node, &tdc->free_dma_desc);  in tegra_dma_abort_all()
540 							&tdc->cb_desc);  in tegra_dma_abort_all()
544 	tdc->isr_handler = NULL;  in tegra_dma_abort_all()
547 static bool handle_continuous_head_request(struct tegra_dma_channel *tdc,  in handle_continuous_head_request()  argument
552 	if (list_empty(&tdc->pending_sg_req)) {  in handle_continuous_head_request()
553 		dev_err(tdc2dev(tdc), "Dma is running without req\n");  in handle_continuous_head_request()
554 		tegra_dma_stop(tdc);  in handle_continuous_head_request()
563 	hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);  in handle_continuous_head_request()
565 		tegra_dma_stop(tdc);  in handle_continuous_head_request()
566 		dev_err(tdc2dev(tdc), "Error in dma transfer, aborting dma\n");  in handle_continuous_head_request()
567 		tegra_dma_abort_all(tdc);  in handle_continuous_head_request()
573 		tdc_configure_next_head_desc(tdc);  in handle_continuous_head_request()
577 static void handle_once_dma_done(struct tegra_dma_channel *tdc,  in handle_once_dma_done()  argument
583 	tdc->busy = false;  in handle_once_dma_done()
584 	sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);  in handle_once_dma_done()
593 			list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);  in handle_once_dma_done()
595 		list_add_tail(&dma_desc->node, &tdc->free_dma_desc);  in handle_once_dma_done()
597 	list_add_tail(&sgreq->node, &tdc->free_sg_req);  in handle_once_dma_done()
600 	if (to_terminate || list_empty(&tdc->pending_sg_req))  in handle_once_dma_done()
603 	tdc_start_head_req(tdc);  in handle_once_dma_done()
607 static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc,  in handle_cont_sngl_cycle_dma_done()  argument
614 	sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);  in handle_cont_sngl_cycle_dma_done()
620 		list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);  in handle_cont_sngl_cycle_dma_done()
624 	if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) {  in handle_cont_sngl_cycle_dma_done()
625 		list_move_tail(&sgreq->node, &tdc->pending_sg_req);  in handle_cont_sngl_cycle_dma_done()
627 		st = handle_continuous_head_request(tdc, sgreq, to_terminate);  in handle_cont_sngl_cycle_dma_done()
636 	struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data;  in tegra_dma_tasklet()  local
643 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_tasklet()
644 	while (!list_empty(&tdc->cb_desc)) {  in tegra_dma_tasklet()
645 		dma_desc  = list_first_entry(&tdc->cb_desc,  in tegra_dma_tasklet()
652 		spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_tasklet()
655 		spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_tasklet()
657 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_tasklet()
662 	struct tegra_dma_channel *tdc = dev_id;  in tegra_dma_isr()  local
666 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_isr()
668 	status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);  in tegra_dma_isr()
670 		tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status);  in tegra_dma_isr()
671 		tdc->isr_handler(tdc, false);  in tegra_dma_isr()
672 		tasklet_schedule(&tdc->tasklet);  in tegra_dma_isr()
673 		spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_isr()
677 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_isr()
678 	dev_info(tdc2dev(tdc),  in tegra_dma_isr()
686 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(txd->chan);  in tegra_dma_tx_submit()  local
690 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_tx_submit()
693 	list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req);  in tegra_dma_tx_submit()
694 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_tx_submit()
700 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);  in tegra_dma_issue_pending()  local
703 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_issue_pending()
704 	if (list_empty(&tdc->pending_sg_req)) {  in tegra_dma_issue_pending()
705 		dev_err(tdc2dev(tdc), "No DMA request\n");  in tegra_dma_issue_pending()
708 	if (!tdc->busy) {  in tegra_dma_issue_pending()
709 		tdc_start_head_req(tdc);  in tegra_dma_issue_pending()
712 		if (tdc->cyclic) {  in tegra_dma_issue_pending()
718 			tdc_configure_next_head_desc(tdc);  in tegra_dma_issue_pending()
722 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_issue_pending()
728 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);  in tegra_dma_terminate_all()  local
736 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_terminate_all()
737 	if (list_empty(&tdc->pending_sg_req)) {  in tegra_dma_terminate_all()
738 		spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_terminate_all()
742 	if (!tdc->busy)  in tegra_dma_terminate_all()
746 	tegra_dma_pause(tdc, true);  in tegra_dma_terminate_all()
748 	status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);  in tegra_dma_terminate_all()
750 		dev_dbg(tdc2dev(tdc), "%s():handling isr\n", __func__);  in tegra_dma_terminate_all()
751 		tdc->isr_handler(tdc, true);  in tegra_dma_terminate_all()
752 		status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);  in tegra_dma_terminate_all()
754 	if (tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_terminate_all()
755 		wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER);  in tegra_dma_terminate_all()
759 	was_busy = tdc->busy;  in tegra_dma_terminate_all()
760 	tegra_dma_stop(tdc);  in tegra_dma_terminate_all()
762 	if (!list_empty(&tdc->pending_sg_req) && was_busy) {  in tegra_dma_terminate_all()
763 		sgreq = list_first_entry(&tdc->pending_sg_req,  in tegra_dma_terminate_all()
766 				get_current_xferred_count(tdc, sgreq, wcount);  in tegra_dma_terminate_all()
768 	tegra_dma_resume(tdc);  in tegra_dma_terminate_all()
771 	tegra_dma_abort_all(tdc);  in tegra_dma_terminate_all()
773 	while (!list_empty(&tdc->cb_desc)) {  in tegra_dma_terminate_all()
774 		dma_desc  = list_first_entry(&tdc->cb_desc,  in tegra_dma_terminate_all()
779 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_terminate_all()
786 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);  in tegra_dma_tx_status()  local
797 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_tx_status()
800 	list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) {  in tegra_dma_tx_status()
807 			spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_tx_status()
813 	list_for_each_entry(sg_req, &tdc->pending_sg_req, node) {  in tegra_dma_tx_status()
821 			spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_tx_status()
826 	dev_dbg(tdc2dev(tdc), "cookie %d does not found\n", cookie);  in tegra_dma_tx_status()
827 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_tx_status()
831 static inline int get_bus_width(struct tegra_dma_channel *tdc,  in get_bus_width()  argument
844 		dev_warn(tdc2dev(tdc),  in get_bus_width()
850 static inline int get_burst_size(struct tegra_dma_channel *tdc,  in get_burst_size()  argument
880 static int get_transfer_param(struct tegra_dma_channel *tdc,  in get_transfer_param()  argument
888 		*apb_addr = tdc->dma_sconfig.dst_addr;  in get_transfer_param()
889 		*apb_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width);  in get_transfer_param()
890 		*burst_size = tdc->dma_sconfig.dst_maxburst;  in get_transfer_param()
891 		*slave_bw = tdc->dma_sconfig.dst_addr_width;  in get_transfer_param()
896 		*apb_addr = tdc->dma_sconfig.src_addr;  in get_transfer_param()
897 		*apb_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width);  in get_transfer_param()
898 		*burst_size = tdc->dma_sconfig.src_maxburst;  in get_transfer_param()
899 		*slave_bw = tdc->dma_sconfig.src_addr_width;  in get_transfer_param()
904 		dev_err(tdc2dev(tdc), "Dma direction is not supported\n");  in get_transfer_param()
910 static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc,  in tegra_dma_prep_wcount()  argument
915 	if (tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_prep_wcount()
926 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);  in tegra_dma_prep_slave_sg()  local
937 	if (!tdc->config_init) {  in tegra_dma_prep_slave_sg()
938 		dev_err(tdc2dev(tdc), "dma channel is not configured\n");  in tegra_dma_prep_slave_sg()
942 		dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len);  in tegra_dma_prep_slave_sg()
946 	ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,  in tegra_dma_prep_slave_sg()
959 	csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;  in tegra_dma_prep_slave_sg()
965 	dma_desc = tegra_dma_desc_get(tdc);  in tegra_dma_prep_slave_sg()
967 		dev_err(tdc2dev(tdc), "Dma descriptors not available\n");  in tegra_dma_prep_slave_sg()
985 				(len > tdc->tdma->chip_data->max_dma_count)) {  in tegra_dma_prep_slave_sg()
986 			dev_err(tdc2dev(tdc),  in tegra_dma_prep_slave_sg()
988 			tegra_dma_desc_put(tdc, dma_desc);  in tegra_dma_prep_slave_sg()
992 		sg_req = tegra_dma_sg_req_get(tdc);  in tegra_dma_prep_slave_sg()
994 			dev_err(tdc2dev(tdc), "Dma sg-req not available\n");  in tegra_dma_prep_slave_sg()
995 			tegra_dma_desc_put(tdc, dma_desc);  in tegra_dma_prep_slave_sg()
999 		ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len);  in tegra_dma_prep_slave_sg()
1005 		tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len);  in tegra_dma_prep_slave_sg()
1023 	if (!tdc->isr_handler) {  in tegra_dma_prep_slave_sg()
1024 		tdc->isr_handler = handle_once_dma_done;  in tegra_dma_prep_slave_sg()
1025 		tdc->cyclic = false;  in tegra_dma_prep_slave_sg()
1027 		if (tdc->cyclic) {  in tegra_dma_prep_slave_sg()
1028 			dev_err(tdc2dev(tdc), "DMA configured in cyclic mode\n");  in tegra_dma_prep_slave_sg()
1029 			tegra_dma_desc_put(tdc, dma_desc);  in tegra_dma_prep_slave_sg()
1042 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);  in tegra_dma_prep_dma_cyclic()  local
1054 		dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");  in tegra_dma_prep_dma_cyclic()
1058 	if (!tdc->config_init) {  in tegra_dma_prep_dma_cyclic()
1059 		dev_err(tdc2dev(tdc), "DMA slave is not configured\n");  in tegra_dma_prep_dma_cyclic()
1069 	if (tdc->busy) {  in tegra_dma_prep_dma_cyclic()
1070 		dev_err(tdc2dev(tdc), "Request not allowed when dma running\n");  in tegra_dma_prep_dma_cyclic()
1079 		dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n");  in tegra_dma_prep_dma_cyclic()
1085 			(len > tdc->tdma->chip_data->max_dma_count)) {  in tegra_dma_prep_dma_cyclic()
1086 		dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");  in tegra_dma_prep_dma_cyclic()
1090 	ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,  in tegra_dma_prep_dma_cyclic()
1104 	csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;  in tegra_dma_prep_dma_cyclic()
1108 	dma_desc = tegra_dma_desc_get(tdc);  in tegra_dma_prep_dma_cyclic()
1110 		dev_err(tdc2dev(tdc), "not enough descriptors available\n");  in tegra_dma_prep_dma_cyclic()
1124 		sg_req = tegra_dma_sg_req_get(tdc);  in tegra_dma_prep_dma_cyclic()
1126 			dev_err(tdc2dev(tdc), "Dma sg-req not available\n");  in tegra_dma_prep_dma_cyclic()
1127 			tegra_dma_desc_put(tdc, dma_desc);  in tegra_dma_prep_dma_cyclic()
1131 		ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len);  in tegra_dma_prep_dma_cyclic()
1135 		tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len);  in tegra_dma_prep_dma_cyclic()
1156 	if (!tdc->isr_handler) {  in tegra_dma_prep_dma_cyclic()
1157 		tdc->isr_handler = handle_cont_sngl_cycle_dma_done;  in tegra_dma_prep_dma_cyclic()
1158 		tdc->cyclic = true;  in tegra_dma_prep_dma_cyclic()
1160 		if (!tdc->cyclic) {  in tegra_dma_prep_dma_cyclic()
1161 			dev_err(tdc2dev(tdc), "DMA configuration conflict\n");  in tegra_dma_prep_dma_cyclic()
1162 			tegra_dma_desc_put(tdc, dma_desc);  in tegra_dma_prep_dma_cyclic()
1172 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);  in tegra_dma_alloc_chan_resources()  local
1173 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_alloc_chan_resources()
1176 	dma_cookie_init(&tdc->dma_chan);  in tegra_dma_alloc_chan_resources()
1177 	tdc->config_init = false;  in tegra_dma_alloc_chan_resources()
1180 		dev_err(tdc2dev(tdc), "clk_prepare_enable failed: %d\n", ret);  in tegra_dma_alloc_chan_resources()
1186 	struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);  in tegra_dma_free_chan_resources()  local
1187 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_free_chan_resources()
1198 	dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);  in tegra_dma_free_chan_resources()
1200 	if (tdc->busy)  in tegra_dma_free_chan_resources()
1203 	spin_lock_irqsave(&tdc->lock, flags);  in tegra_dma_free_chan_resources()
1204 	list_splice_init(&tdc->pending_sg_req, &sg_req_list);  in tegra_dma_free_chan_resources()
1205 	list_splice_init(&tdc->free_sg_req, &sg_req_list);  in tegra_dma_free_chan_resources()
1206 	list_splice_init(&tdc->free_dma_desc, &dma_desc_list);  in tegra_dma_free_chan_resources()
1207 	INIT_LIST_HEAD(&tdc->cb_desc);  in tegra_dma_free_chan_resources()
1208 	tdc->config_init = false;  in tegra_dma_free_chan_resources()
1209 	tdc->isr_handler = NULL;  in tegra_dma_free_chan_resources()
1210 	spin_unlock_irqrestore(&tdc->lock, flags);  in tegra_dma_free_chan_resources()
1226 	tdc->slave_id = 0;  in tegra_dma_free_chan_resources()
1234 	struct tegra_dma_channel *tdc;  in tegra_dma_of_xlate()  local
1240 	tdc = to_tegra_dma_chan(chan);  in tegra_dma_of_xlate()
1241 	tdc->slave_id = dma_spec->args[0];  in tegra_dma_of_xlate()
1378 		struct tegra_dma_channel *tdc = &tdma->channels[i];  in tegra_dma_probe()  local
1380 		tdc->chan_base_offset = TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET +  in tegra_dma_probe()
1389 		tdc->irq = res->start;  in tegra_dma_probe()
1390 		snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i);  in tegra_dma_probe()
1391 		ret = devm_request_irq(&pdev->dev, tdc->irq,  in tegra_dma_probe()
1392 				tegra_dma_isr, 0, tdc->name, tdc);  in tegra_dma_probe()
1400 		tdc->dma_chan.device = &tdma->dma_dev;  in tegra_dma_probe()
1401 		dma_cookie_init(&tdc->dma_chan);  in tegra_dma_probe()
1402 		list_add_tail(&tdc->dma_chan.device_node,  in tegra_dma_probe()
1404 		tdc->tdma = tdma;  in tegra_dma_probe()
1405 		tdc->id = i;  in tegra_dma_probe()
1407 		tasklet_init(&tdc->tasklet, tegra_dma_tasklet,  in tegra_dma_probe()
1408 				(unsigned long)tdc);  in tegra_dma_probe()
1409 		spin_lock_init(&tdc->lock);  in tegra_dma_probe()
1411 		INIT_LIST_HEAD(&tdc->pending_sg_req);  in tegra_dma_probe()
1412 		INIT_LIST_HEAD(&tdc->free_sg_req);  in tegra_dma_probe()
1413 		INIT_LIST_HEAD(&tdc->free_dma_desc);  in tegra_dma_probe()
1414 		INIT_LIST_HEAD(&tdc->cb_desc);  in tegra_dma_probe()
1471 		struct tegra_dma_channel *tdc = &tdma->channels[i];  in tegra_dma_probe()  local
1472 		tasklet_kill(&tdc->tasklet);  in tegra_dma_probe()
1486 	struct tegra_dma_channel *tdc;  in tegra_dma_remove()  local
1491 		tdc = &tdma->channels[i];  in tegra_dma_remove()
1492 		tasklet_kill(&tdc->tasklet);  in tegra_dma_remove()
1539 		struct tegra_dma_channel *tdc = &tdma->channels[i];  in tegra_dma_pm_suspend()  local
1540 		struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;  in tegra_dma_pm_suspend()
1542 		ch_reg->csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);  in tegra_dma_pm_suspend()
1543 		ch_reg->ahb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBPTR);  in tegra_dma_pm_suspend()
1544 		ch_reg->apb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBPTR);  in tegra_dma_pm_suspend()
1545 		ch_reg->ahb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBSEQ);  in tegra_dma_pm_suspend()
1546 		ch_reg->apb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBSEQ);  in tegra_dma_pm_suspend()
1570 		struct tegra_dma_channel *tdc = &tdma->channels[i];  in tegra_dma_pm_resume()  local
1571 		struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;  in tegra_dma_pm_resume()
1573 		tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_reg->apb_seq);  in tegra_dma_pm_resume()
1574 		tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_reg->apb_ptr);  in tegra_dma_pm_resume()
1575 		tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_reg->ahb_seq);  in tegra_dma_pm_resume()
1576 		tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_reg->ahb_ptr);  in tegra_dma_pm_resume()
1577 		tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,  in tegra_dma_pm_resume()