261 static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan,  in s3c24xx_dma_phy_valid()  argument
264 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_phy_valid()
266 	struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];  in s3c24xx_dma_phy_valid()
270 	if (!s3cchan->slave)  in s3c24xx_dma_phy_valid()
289 struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan)  in s3c24xx_dma_get_phy()  argument
291 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_get_phy()
299 	if (s3cchan->slave)  in s3c24xx_dma_get_phy()
300 		cdata = &pdata->channels[s3cchan->id];  in s3c24xx_dma_get_phy()
308 		if (!s3c24xx_dma_phy_valid(s3cchan, phy))  in s3c24xx_dma_get_phy()
314 			phy->serving = s3cchan;  in s3c24xx_dma_get_phy()
378 static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan)  in s3c24xx_dma_getbytes_chan()  argument
380 	struct s3c24xx_dma_phy *phy = s3cchan->phy;  in s3c24xx_dma_getbytes_chan()
381 	struct s3c24xx_txd *txd = s3cchan->at;  in s3c24xx_dma_getbytes_chan()
390 	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);  in s3c24xx_dma_set_runtime_config()  local
399 	spin_lock_irqsave(&s3cchan->vc.lock, flags);  in s3c24xx_dma_set_runtime_config()
401 	if (!s3cchan->slave) {  in s3c24xx_dma_set_runtime_config()
406 	s3cchan->cfg = *config;  in s3c24xx_dma_set_runtime_config()
409 	spin_unlock_irqrestore(&s3cchan->vc.lock, flags);  in s3c24xx_dma_set_runtime_config()
447 static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan,  in s3c24xx_dma_start_next_sg()  argument
450 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_start_next_sg()
451 	struct s3c24xx_dma_phy *phy = s3cchan->phy;  in s3c24xx_dma_start_next_sg()
470 	if (s3cchan->slave) {  in s3c24xx_dma_start_next_sg()
472 					&pdata->channels[s3cchan->id];  in s3c24xx_dma_start_next_sg()
502 	if (!s3cchan->slave)  in s3c24xx_dma_start_next_sg()
511 static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan)  in s3c24xx_dma_start_next_txd()  argument
513 	struct s3c24xx_dma_phy *phy = s3cchan->phy;  in s3c24xx_dma_start_next_txd()
514 	struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc);  in s3c24xx_dma_start_next_txd()
519 	s3cchan->at = txd;  in s3c24xx_dma_start_next_txd()
527 	s3c24xx_dma_start_next_sg(s3cchan, txd);  in s3c24xx_dma_start_next_txd()
531 				struct s3c24xx_dma_chan *s3cchan)  in s3c24xx_dma_free_txd_list()  argument
535 	vchan_get_all_descriptors(&s3cchan->vc, &head);  in s3c24xx_dma_free_txd_list()
536 	vchan_dma_desc_free_list(&s3cchan->vc, &head);  in s3c24xx_dma_free_txd_list()
544 static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan)  in s3c24xx_dma_phy_alloc_and_start()  argument
546 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_phy_alloc_and_start()
549 	phy = s3c24xx_dma_get_phy(s3cchan);  in s3c24xx_dma_phy_alloc_and_start()
552 			s3cchan->name);  in s3c24xx_dma_phy_alloc_and_start()
553 		s3cchan->state = S3C24XX_DMA_CHAN_WAITING;  in s3c24xx_dma_phy_alloc_and_start()
558 		phy->id, s3cchan->name);  in s3c24xx_dma_phy_alloc_and_start()
560 	s3cchan->phy = phy;  in s3c24xx_dma_phy_alloc_and_start()
561 	s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;  in s3c24xx_dma_phy_alloc_and_start()
563 	s3c24xx_dma_start_next_txd(s3cchan);  in s3c24xx_dma_phy_alloc_and_start()
567 	struct s3c24xx_dma_chan *s3cchan)  in s3c24xx_dma_phy_reassign_start()  argument
569 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_phy_reassign_start()
572 		phy->id, s3cchan->name);  in s3c24xx_dma_phy_reassign_start()
579 	phy->serving = s3cchan;  in s3c24xx_dma_phy_reassign_start()
580 	s3cchan->phy = phy;  in s3c24xx_dma_phy_reassign_start()
581 	s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;  in s3c24xx_dma_phy_reassign_start()
582 	s3c24xx_dma_start_next_txd(s3cchan);  in s3c24xx_dma_phy_reassign_start()
589 static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan)  in s3c24xx_dma_phy_free()  argument
591 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_phy_free()
608 				      s3c24xx_dma_phy_valid(p, s3cchan->phy)) {  in s3c24xx_dma_phy_free()
615 	s3c24xx_dma_terminate_phy(s3cchan->phy);  in s3c24xx_dma_phy_free()
628 			s3c24xx_dma_phy_reassign_start(s3cchan->phy, next);  in s3c24xx_dma_phy_free()
636 		s3c24xx_dma_put_phy(s3cchan->phy);  in s3c24xx_dma_phy_free()
639 	s3cchan->phy = NULL;  in s3c24xx_dma_phy_free()
640 	s3cchan->state = S3C24XX_DMA_CHAN_IDLE;  in s3c24xx_dma_phy_free()
646 	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan);  in s3c24xx_dma_desc_free()  local
648 	if (!s3cchan->slave)  in s3c24xx_dma_desc_free()
657 	struct s3c24xx_dma_chan *s3cchan = phy->serving;  in s3c24xx_dma_irq()  local
669 	if (unlikely(!s3cchan)) {  in s3c24xx_dma_irq()
678 	spin_lock(&s3cchan->vc.lock);  in s3c24xx_dma_irq()
679 	txd = s3cchan->at;  in s3c24xx_dma_irq()
686 			s3c24xx_dma_start_next_sg(s3cchan, txd);  in s3c24xx_dma_irq()
688 			s3cchan->at = NULL;  in s3c24xx_dma_irq()
695 			if (vchan_next_desc(&s3cchan->vc))  in s3c24xx_dma_irq()
696 				s3c24xx_dma_start_next_txd(s3cchan);  in s3c24xx_dma_irq()
698 				s3c24xx_dma_phy_free(s3cchan);  in s3c24xx_dma_irq()
704 			s3c24xx_dma_start_next_sg(s3cchan, txd);  in s3c24xx_dma_irq()
707 	spin_unlock(&s3cchan->vc.lock);  in s3c24xx_dma_irq()
718 	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);  in s3c24xx_dma_terminate_all()  local
719 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_terminate_all()
723 	spin_lock_irqsave(&s3cchan->vc.lock, flags);  in s3c24xx_dma_terminate_all()
725 	if (!s3cchan->phy && !s3cchan->at) {  in s3c24xx_dma_terminate_all()
727 			s3cchan->id);  in s3c24xx_dma_terminate_all()
732 	s3cchan->state = S3C24XX_DMA_CHAN_IDLE;  in s3c24xx_dma_terminate_all()
735 	if (s3cchan->phy)  in s3c24xx_dma_terminate_all()
736 		s3c24xx_dma_phy_free(s3cchan);  in s3c24xx_dma_terminate_all()
739 	if (s3cchan->at) {  in s3c24xx_dma_terminate_all()
740 		s3c24xx_dma_desc_free(&s3cchan->at->vd);  in s3c24xx_dma_terminate_all()
741 		s3cchan->at = NULL;  in s3c24xx_dma_terminate_all()
745 	s3c24xx_dma_free_txd_list(s3cdma, s3cchan);  in s3c24xx_dma_terminate_all()
747 	spin_unlock_irqrestore(&s3cchan->vc.lock, flags);  in s3c24xx_dma_terminate_all()
761 	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);  in s3c24xx_dma_tx_status()  local
769 	spin_lock_irqsave(&s3cchan->vc.lock, flags);  in s3c24xx_dma_tx_status()
772 		spin_unlock_irqrestore(&s3cchan->vc.lock, flags);  in s3c24xx_dma_tx_status()
781 		spin_unlock_irqrestore(&s3cchan->vc.lock, flags);  in s3c24xx_dma_tx_status()
785 	vd = vchan_find_desc(&s3cchan->vc, cookie);  in s3c24xx_dma_tx_status()
797 		txd = s3cchan->at;  in s3c24xx_dma_tx_status()
803 		bytes += s3c24xx_dma_getbytes_chan(s3cchan);  in s3c24xx_dma_tx_status()
805 	spin_unlock_irqrestore(&s3cchan->vc.lock, flags);  in s3c24xx_dma_tx_status()
824 	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);  in s3c24xx_dma_prep_memcpy()  local
825 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_prep_memcpy()
831 			len, s3cchan->name);  in s3c24xx_dma_prep_memcpy()
879 	return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);  in s3c24xx_dma_prep_memcpy()
886 	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);  in s3c24xx_dma_prep_dma_cyclic()  local
887 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_prep_dma_cyclic()
889 	struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];  in s3c24xx_dma_prep_dma_cyclic()
899 		size, period, s3cchan->name);  in s3c24xx_dma_prep_dma_cyclic()
943 		slave_addr = s3cchan->cfg.dst_addr;  in s3c24xx_dma_prep_dma_cyclic()
944 		txd->width = s3cchan->cfg.dst_addr_width;  in s3c24xx_dma_prep_dma_cyclic()
949 		slave_addr = s3cchan->cfg.src_addr;  in s3c24xx_dma_prep_dma_cyclic()
950 		txd->width = s3cchan->cfg.src_addr_width;  in s3c24xx_dma_prep_dma_cyclic()
976 	return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);  in s3c24xx_dma_prep_dma_cyclic()
984 	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);  in s3c24xx_dma_prep_slave_sg()  local
985 	struct s3c24xx_dma_engine *s3cdma = s3cchan->host;  in s3c24xx_dma_prep_slave_sg()
987 	struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];  in s3c24xx_dma_prep_slave_sg()
996 			sg_dma_len(sgl), s3cchan->name);  in s3c24xx_dma_prep_slave_sg()
1032 		slave_addr = s3cchan->cfg.dst_addr;  in s3c24xx_dma_prep_slave_sg()
1033 		txd->width = s3cchan->cfg.dst_addr_width;  in s3c24xx_dma_prep_slave_sg()
1038 		slave_addr = s3cchan->cfg.src_addr;  in s3c24xx_dma_prep_slave_sg()
1039 		txd->width = s3cchan->cfg.src_addr_width;  in s3c24xx_dma_prep_slave_sg()
1065 	return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);  in s3c24xx_dma_prep_slave_sg()
1074 	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);  in s3c24xx_dma_issue_pending()  local
1077 	spin_lock_irqsave(&s3cchan->vc.lock, flags);  in s3c24xx_dma_issue_pending()
1078 	if (vchan_issue_pending(&s3cchan->vc)) {  in s3c24xx_dma_issue_pending()
1079 		if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING)  in s3c24xx_dma_issue_pending()
1080 			s3c24xx_dma_phy_alloc_and_start(s3cchan);  in s3c24xx_dma_issue_pending()
1082 	spin_unlock_irqrestore(&s3cchan->vc.lock, flags);  in s3c24xx_dma_issue_pending()
1405 	struct s3c24xx_dma_chan *s3cchan;  in s3c24xx_dma_filter()  local
1410 	s3cchan = to_s3c24xx_dma_chan(chan);  in s3c24xx_dma_filter()
1412 	return s3cchan->id == (int)param;  in s3c24xx_dma_filter()