105 static bool dw_mci_reset(struct dw_mci *host);
106 static bool dw_mci_ctrl_reset(struct dw_mci *host, u32 reset);
119 	spin_lock_bh(&slot->host->lock);  in dw_mci_req_show()
145 	spin_unlock_bh(&slot->host->lock);  in dw_mci_req_show()
191 	struct dw_mci *host = slot->host;  in dw_mci_init_debugfs()  local
199 	node = debugfs_create_file("regs", S_IRUSR, root, host,  in dw_mci_init_debugfs()
209 	node = debugfs_create_u32("state", S_IRUSR, root, (u32 *)&host->state);  in dw_mci_init_debugfs()
214 				  (u32 *)&host->pending_events);  in dw_mci_init_debugfs()
219 				  (u32 *)&host->completed_events);  in dw_mci_init_debugfs()
236 	struct dw_mci *host = slot->host;  in dw_mci_prepare_command()  local
237 	const struct dw_mci_drv_data *drv_data = slot->host->drv_data;  in dw_mci_prepare_command()
259 		WARN_ON(slot->host->state != STATE_SENDING_CMD);  in dw_mci_prepare_command()
260 		slot->host->state = STATE_SENDING_CMD11;  in dw_mci_prepare_command()
273 		clk_en_a = mci_readl(host, CLKENA);  in dw_mci_prepare_command()
275 		mci_writel(host, CLKENA, clk_en_a);  in dw_mci_prepare_command()
300 		drv_data->prepare_command(slot->host, &cmdr);  in dw_mci_prepare_command()
305 static u32 dw_mci_prep_stop_abort(struct dw_mci *host, struct mmc_command *cmd)  in dw_mci_prep_stop_abort()  argument
313 	stop = &host->stop_abort;  in dw_mci_prep_stop_abort()
341 static void dw_mci_wait_while_busy(struct dw_mci *host, u32 cmd_flags)  in dw_mci_wait_while_busy()  argument
355 		while (mci_readl(host, STATUS) & SDMMC_STATUS_BUSY) {  in dw_mci_wait_while_busy()
358 				dev_err(host->dev, "Busy; trying anyway\n");  in dw_mci_wait_while_busy()
366 static void dw_mci_start_command(struct dw_mci *host,  in dw_mci_start_command()  argument
369 	host->cmd = cmd;  in dw_mci_start_command()
370 	dev_vdbg(host->dev,  in dw_mci_start_command()
374 	mci_writel(host, CMDARG, cmd->arg);  in dw_mci_start_command()
376 	dw_mci_wait_while_busy(host, cmd_flags);  in dw_mci_start_command()
378 	mci_writel(host, CMD, cmd_flags | SDMMC_CMD_START);  in dw_mci_start_command()
381 static inline void send_stop_abort(struct dw_mci *host, struct mmc_data *data)  in send_stop_abort()  argument
383 	struct mmc_command *stop = data->stop ? data->stop : &host->stop_abort;  in send_stop_abort()
385 	dw_mci_start_command(host, stop, host->stop_cmdr);  in send_stop_abort()
389 static void dw_mci_stop_dma(struct dw_mci *host)  in dw_mci_stop_dma()  argument
391 	if (host->using_dma) {  in dw_mci_stop_dma()
392 		host->dma_ops->stop(host);  in dw_mci_stop_dma()
393 		host->dma_ops->cleanup(host);  in dw_mci_stop_dma()
397 	set_bit(EVENT_XFER_COMPLETE, &host->pending_events);  in dw_mci_stop_dma()
408 static void dw_mci_dma_cleanup(struct dw_mci *host)  in dw_mci_dma_cleanup()  argument
410 	struct mmc_data *data = host->data;  in dw_mci_dma_cleanup()
414 			dma_unmap_sg(host->dev,  in dw_mci_dma_cleanup()
420 static void dw_mci_idmac_reset(struct dw_mci *host)  in dw_mci_idmac_reset()  argument
422 	u32 bmod = mci_readl(host, BMOD);  in dw_mci_idmac_reset()
425 	mci_writel(host, BMOD, bmod);  in dw_mci_idmac_reset()
428 static void dw_mci_idmac_stop_dma(struct dw_mci *host)  in dw_mci_idmac_stop_dma()  argument
433 	temp = mci_readl(host, CTRL);  in dw_mci_idmac_stop_dma()
436 	mci_writel(host, CTRL, temp);  in dw_mci_idmac_stop_dma()
439 	temp = mci_readl(host, BMOD);  in dw_mci_idmac_stop_dma()
442 	mci_writel(host, BMOD, temp);  in dw_mci_idmac_stop_dma()
447 	struct dw_mci *host = arg;  in dw_mci_dmac_complete_dma()  local
448 	struct mmc_data *data = host->data;  in dw_mci_dmac_complete_dma()
450 	dev_vdbg(host->dev, "DMA complete\n");  in dw_mci_dmac_complete_dma()
452 	if ((host->use_dma == TRANS_MODE_EDMAC) &&  in dw_mci_dmac_complete_dma()
455 		dma_sync_sg_for_cpu(mmc_dev(host->cur_slot->mmc),  in dw_mci_dmac_complete_dma()
460 	host->dma_ops->cleanup(host);  in dw_mci_dmac_complete_dma()
467 		set_bit(EVENT_XFER_COMPLETE, &host->pending_events);  in dw_mci_dmac_complete_dma()
468 		tasklet_schedule(&host->tasklet);  in dw_mci_dmac_complete_dma()
472 static void dw_mci_translate_sglist(struct dw_mci *host, struct mmc_data *data,  in dw_mci_translate_sglist()  argument
478 	if (host->dma_64bit_address == 1) {  in dw_mci_translate_sglist()
481 		desc_first = desc_last = desc = host->sg_cpu;  in dw_mci_translate_sglist()
526 		desc_first = desc_last = desc = host->sg_cpu;  in dw_mci_translate_sglist()
573 static int dw_mci_idmac_start_dma(struct dw_mci *host, unsigned int sg_len)  in dw_mci_idmac_start_dma()  argument
577 	dw_mci_translate_sglist(host, host->data, sg_len);  in dw_mci_idmac_start_dma()
580 	dw_mci_ctrl_reset(host, SDMMC_CTRL_DMA_RESET);  in dw_mci_idmac_start_dma()
581 	dw_mci_idmac_reset(host);  in dw_mci_idmac_start_dma()
584 	temp = mci_readl(host, CTRL);  in dw_mci_idmac_start_dma()
586 	mci_writel(host, CTRL, temp);  in dw_mci_idmac_start_dma()
592 	temp = mci_readl(host, BMOD);  in dw_mci_idmac_start_dma()
594 	mci_writel(host, BMOD, temp);  in dw_mci_idmac_start_dma()
597 	mci_writel(host, PLDMND, 1);  in dw_mci_idmac_start_dma()
602 static int dw_mci_idmac_init(struct dw_mci *host)  in dw_mci_idmac_init()  argument
606 	if (host->dma_64bit_address == 1) {  in dw_mci_idmac_init()
609 		host->ring_size = PAGE_SIZE / sizeof(struct idmac_desc_64addr);  in dw_mci_idmac_init()
612 		for (i = 0, p = host->sg_cpu; i < host->ring_size - 1;  in dw_mci_idmac_init()
614 			p->des6 = (host->sg_dma +  in dw_mci_idmac_init()
618 			p->des7 = (u64)(host->sg_dma +  in dw_mci_idmac_init()
628 		p->des6 = host->sg_dma & 0xffffffff;  in dw_mci_idmac_init()
629 		p->des7 = (u64)host->sg_dma >> 32;  in dw_mci_idmac_init()
635 		host->ring_size = PAGE_SIZE / sizeof(struct idmac_desc);  in dw_mci_idmac_init()
638 		for (i = 0, p = host->sg_cpu;  in dw_mci_idmac_init()
639 		     i < host->ring_size - 1;  in dw_mci_idmac_init()
641 			p->des3 = cpu_to_le32(host->sg_dma +  in dw_mci_idmac_init()
647 		p->des3 = cpu_to_le32(host->sg_dma);  in dw_mci_idmac_init()
651 	dw_mci_idmac_reset(host);  in dw_mci_idmac_init()
653 	if (host->dma_64bit_address == 1) {  in dw_mci_idmac_init()
655 		mci_writel(host, IDSTS64, IDMAC_INT_CLR);  in dw_mci_idmac_init()
656 		mci_writel(host, IDINTEN64, SDMMC_IDMAC_INT_NI |  in dw_mci_idmac_init()
660 		mci_writel(host, DBADDRL, host->sg_dma & 0xffffffff);  in dw_mci_idmac_init()
661 		mci_writel(host, DBADDRU, (u64)host->sg_dma >> 32);  in dw_mci_idmac_init()
665 		mci_writel(host, IDSTS, IDMAC_INT_CLR);  in dw_mci_idmac_init()
666 		mci_writel(host, IDINTEN, SDMMC_IDMAC_INT_NI |  in dw_mci_idmac_init()
670 		mci_writel(host, DBADDR, host->sg_dma);  in dw_mci_idmac_init()
684 static void dw_mci_edmac_stop_dma(struct dw_mci *host)  in dw_mci_edmac_stop_dma()  argument
686 	dmaengine_terminate_all(host->dms->ch);  in dw_mci_edmac_stop_dma()
689 static int dw_mci_edmac_start_dma(struct dw_mci *host,  in dw_mci_edmac_start_dma()  argument
694 	struct scatterlist *sgl = host->data->sg;  in dw_mci_edmac_start_dma()
696 	u32 sg_elems = host->data->sg_len;  in dw_mci_edmac_start_dma()
698 	u32 fifo_offset = host->fifo_reg - host->regs;  in dw_mci_edmac_start_dma()
702 	cfg.dst_addr = (dma_addr_t)(host->phy_regs + fifo_offset);  in dw_mci_edmac_start_dma()
708 	fifoth_val = mci_readl(host, FIFOTH);  in dw_mci_edmac_start_dma()
712 	if (host->data->flags & MMC_DATA_WRITE)  in dw_mci_edmac_start_dma()
717 	ret = dmaengine_slave_config(host->dms->ch, &cfg);  in dw_mci_edmac_start_dma()
719 		dev_err(host->dev, "Failed to config edmac.\n");  in dw_mci_edmac_start_dma()
723 	desc = dmaengine_prep_slave_sg(host->dms->ch, sgl,  in dw_mci_edmac_start_dma()
727 		dev_err(host->dev, "Can't prepare slave sg.\n");  in dw_mci_edmac_start_dma()
733 	desc->callback_param = (void *)host;  in dw_mci_edmac_start_dma()
737 	if (host->data->flags & MMC_DATA_WRITE)  in dw_mci_edmac_start_dma()
738 		dma_sync_sg_for_device(mmc_dev(host->cur_slot->mmc), sgl,  in dw_mci_edmac_start_dma()
741 	dma_async_issue_pending(host->dms->ch);  in dw_mci_edmac_start_dma()
746 static int dw_mci_edmac_init(struct dw_mci *host)  in dw_mci_edmac_init()  argument
749 	host->dms = kzalloc(sizeof(struct dw_mci_dma_slave), GFP_KERNEL);  in dw_mci_edmac_init()
750 	if (!host->dms)  in dw_mci_edmac_init()
753 	host->dms->ch = dma_request_slave_channel(host->dev, "rx-tx");  in dw_mci_edmac_init()
754 	if (!host->dms->ch) {  in dw_mci_edmac_init()
755 		dev_err(host->dev, "Failed to get external DMA channel.\n");  in dw_mci_edmac_init()
756 		kfree(host->dms);  in dw_mci_edmac_init()
757 		host->dms = NULL;  in dw_mci_edmac_init()
764 static void dw_mci_edmac_exit(struct dw_mci *host)  in dw_mci_edmac_exit()  argument
766 	if (host->dms) {  in dw_mci_edmac_exit()
767 		if (host->dms->ch) {  in dw_mci_edmac_exit()
768 			dma_release_channel(host->dms->ch);  in dw_mci_edmac_exit()
769 			host->dms->ch = NULL;  in dw_mci_edmac_exit()
771 		kfree(host->dms);  in dw_mci_edmac_exit()
772 		host->dms = NULL;  in dw_mci_edmac_exit()
785 static int dw_mci_pre_dma_transfer(struct dw_mci *host,  in dw_mci_pre_dma_transfer()  argument
811 	sg_len = dma_map_sg(host->dev,  in dw_mci_pre_dma_transfer()
831 	if (!slot->host->use_dma || !data)  in dw_mci_pre_req()
839 	if (dw_mci_pre_dma_transfer(slot->host, mrq->data, 1) < 0)  in dw_mci_pre_req()
850 	if (!slot->host->use_dma || !data)  in dw_mci_post_req()
854 		dma_unmap_sg(slot->host->dev,  in dw_mci_post_req()
861 static void dw_mci_adjust_fifoth(struct dw_mci *host, struct mmc_data *data)  in dw_mci_adjust_fifoth()  argument
865 	u32 fifo_width = 1 << host->data_shift;  in dw_mci_adjust_fifoth()
871 	if (!host->use_dma)  in dw_mci_adjust_fifoth()
874 	tx_wmark = (host->fifo_depth) / 2;  in dw_mci_adjust_fifoth()
875 	tx_wmark_invers = host->fifo_depth - tx_wmark;  in dw_mci_adjust_fifoth()
901 	mci_writel(host, FIFOTH, fifoth_val);  in dw_mci_adjust_fifoth()
904 static void dw_mci_ctrl_rd_thld(struct dw_mci *host, struct mmc_data *data)  in dw_mci_ctrl_rd_thld()  argument
916 	if (host->verid < DW_MMC_240A)  in dw_mci_ctrl_rd_thld()
919 	if (host->timing != MMC_TIMING_MMC_HS200 &&  in dw_mci_ctrl_rd_thld()
920 	    host->timing != MMC_TIMING_MMC_HS400 &&  in dw_mci_ctrl_rd_thld()
921 	    host->timing != MMC_TIMING_UHS_SDR104)  in dw_mci_ctrl_rd_thld()
924 	blksz_depth = blksz / (1 << host->data_shift);  in dw_mci_ctrl_rd_thld()
925 	fifo_depth = host->fifo_depth;  in dw_mci_ctrl_rd_thld()
936 	mci_writel(host, CDTHRCTL, SDMMC_SET_RD_THLD(thld_size, 1));  in dw_mci_ctrl_rd_thld()
940 	mci_writel(host, CDTHRCTL, SDMMC_SET_RD_THLD(0, 0));  in dw_mci_ctrl_rd_thld()
943 static int dw_mci_submit_data_dma(struct dw_mci *host, struct mmc_data *data)  in dw_mci_submit_data_dma()  argument
949 	host->using_dma = 0;  in dw_mci_submit_data_dma()
952 	if (!host->use_dma)  in dw_mci_submit_data_dma()
955 	sg_len = dw_mci_pre_dma_transfer(host, data, 0);  in dw_mci_submit_data_dma()
957 		host->dma_ops->stop(host);  in dw_mci_submit_data_dma()
961 	host->using_dma = 1;  in dw_mci_submit_data_dma()
963 	if (host->use_dma == TRANS_MODE_IDMAC)  in dw_mci_submit_data_dma()
964 		dev_vdbg(host->dev,  in dw_mci_submit_data_dma()
966 			 (unsigned long)host->sg_cpu,  in dw_mci_submit_data_dma()
967 			 (unsigned long)host->sg_dma,  in dw_mci_submit_data_dma()
975 	if (host->prev_blksz != data->blksz)  in dw_mci_submit_data_dma()
976 		dw_mci_adjust_fifoth(host, data);  in dw_mci_submit_data_dma()
979 	temp = mci_readl(host, CTRL);  in dw_mci_submit_data_dma()
981 	mci_writel(host, CTRL, temp);  in dw_mci_submit_data_dma()
984 	spin_lock_irqsave(&host->irq_lock, irqflags);  in dw_mci_submit_data_dma()
985 	temp = mci_readl(host, INTMASK);  in dw_mci_submit_data_dma()
987 	mci_writel(host, INTMASK, temp);  in dw_mci_submit_data_dma()
988 	spin_unlock_irqrestore(&host->irq_lock, irqflags);  in dw_mci_submit_data_dma()
990 	if (host->dma_ops->start(host, sg_len)) {  in dw_mci_submit_data_dma()
992 		dev_err(host->dev, "%s: failed to start DMA.\n", __func__);  in dw_mci_submit_data_dma()
999 static void dw_mci_submit_data(struct dw_mci *host, struct mmc_data *data)  in dw_mci_submit_data()  argument
1007 	WARN_ON(host->data);  in dw_mci_submit_data()
1008 	host->sg = NULL;  in dw_mci_submit_data()
1009 	host->data = data;  in dw_mci_submit_data()
1012 		host->dir_status = DW_MCI_RECV_STATUS;  in dw_mci_submit_data()
1013 		dw_mci_ctrl_rd_thld(host, data);  in dw_mci_submit_data()
1015 		host->dir_status = DW_MCI_SEND_STATUS;  in dw_mci_submit_data()
1018 	if (dw_mci_submit_data_dma(host, data)) {  in dw_mci_submit_data()
1019 		if (host->data->flags & MMC_DATA_READ)  in dw_mci_submit_data()
1024 		sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);  in dw_mci_submit_data()
1025 		host->sg = data->sg;  in dw_mci_submit_data()
1026 		host->part_buf_start = 0;  in dw_mci_submit_data()
1027 		host->part_buf_count = 0;  in dw_mci_submit_data()
1029 		mci_writel(host, RINTSTS, SDMMC_INT_TXDR | SDMMC_INT_RXDR);  in dw_mci_submit_data()
1031 		spin_lock_irqsave(&host->irq_lock, irqflags);  in dw_mci_submit_data()
1032 		temp = mci_readl(host, INTMASK);  in dw_mci_submit_data()
1034 		mci_writel(host, INTMASK, temp);  in dw_mci_submit_data()
1035 		spin_unlock_irqrestore(&host->irq_lock, irqflags);  in dw_mci_submit_data()
1037 		temp = mci_readl(host, CTRL);  in dw_mci_submit_data()
1039 		mci_writel(host, CTRL, temp);  in dw_mci_submit_data()
1046 		mci_writel(host, FIFOTH, host->fifoth_val);  in dw_mci_submit_data()
1047 		host->prev_blksz = 0;  in dw_mci_submit_data()
1054 		host->prev_blksz = data->blksz;  in dw_mci_submit_data()
1060 	struct dw_mci *host = slot->host;  in mci_send_cmd()  local
1064 	mci_writel(host, CMDARG, arg);  in mci_send_cmd()
1066 	dw_mci_wait_while_busy(host, cmd);  in mci_send_cmd()
1067 	mci_writel(host, CMD, SDMMC_CMD_START | cmd);  in mci_send_cmd()
1070 		cmd_status = mci_readl(host, CMD);  in mci_send_cmd()
1081 	struct dw_mci *host = slot->host;  in dw_mci_setup_bus()  local
1088 	if (host->state == STATE_WAITING_CMD11_DONE)  in dw_mci_setup_bus()
1092 		mci_writel(host, CLKENA, 0);  in dw_mci_setup_bus()
1094 	} else if (clock != host->current_speed || force_clkinit) {  in dw_mci_setup_bus()
1095 		div = host->bus_hz / clock;  in dw_mci_setup_bus()
1096 		if (host->bus_hz % clock && host->bus_hz > clock)  in dw_mci_setup_bus()
1103 		div = (host->bus_hz != clock) ? DIV_ROUND_UP(div, 2) : 0;  in dw_mci_setup_bus()
1108 				 slot->id, host->bus_hz, clock,  in dw_mci_setup_bus()
1109 				 div ? ((host->bus_hz / div) >> 1) :  in dw_mci_setup_bus()
1110 				 host->bus_hz, div);  in dw_mci_setup_bus()
1113 		mci_writel(host, CLKENA, 0);  in dw_mci_setup_bus()
1114 		mci_writel(host, CLKSRC, 0);  in dw_mci_setup_bus()
1120 		mci_writel(host, CLKDIV, div);  in dw_mci_setup_bus()
1129 		mci_writel(host, CLKENA, clk_en_a);  in dw_mci_setup_bus()
1138 	host->current_speed = clock;  in dw_mci_setup_bus()
1141 	mci_writel(host, CTYPE, (slot->ctype << slot->id));  in dw_mci_setup_bus()
1144 static void __dw_mci_start_request(struct dw_mci *host,  in __dw_mci_start_request()  argument
1154 	host->cur_slot = slot;  in __dw_mci_start_request()
1155 	host->mrq = mrq;  in __dw_mci_start_request()
1157 	host->pending_events = 0;  in __dw_mci_start_request()
1158 	host->completed_events = 0;  in __dw_mci_start_request()
1159 	host->cmd_status = 0;  in __dw_mci_start_request()
1160 	host->data_status = 0;  in __dw_mci_start_request()
1161 	host->dir_status = 0;  in __dw_mci_start_request()
1165 		mci_writel(host, TMOUT, 0xFFFFFFFF);  in __dw_mci_start_request()
1166 		mci_writel(host, BYTCNT, data->blksz*data->blocks);  in __dw_mci_start_request()
1167 		mci_writel(host, BLKSIZ, data->blksz);  in __dw_mci_start_request()
1177 		dw_mci_submit_data(host, data);  in __dw_mci_start_request()
1181 	dw_mci_start_command(host, cmd, cmdflags);  in __dw_mci_start_request()
1196 		spin_lock_irqsave(&host->irq_lock, irqflags);  in __dw_mci_start_request()
1197 		if (!test_bit(EVENT_CMD_COMPLETE, &host->pending_events))  in __dw_mci_start_request()
1198 			mod_timer(&host->cmd11_timer,  in __dw_mci_start_request()
1200 		spin_unlock_irqrestore(&host->irq_lock, irqflags);  in __dw_mci_start_request()
1204 		host->stop_cmdr = dw_mci_prepare_command(slot->mmc, mrq->stop);  in __dw_mci_start_request()
1206 		host->stop_cmdr = dw_mci_prep_stop_abort(host, cmd);  in __dw_mci_start_request()
1209 static void dw_mci_start_request(struct dw_mci *host,  in dw_mci_start_request()  argument
1216 	__dw_mci_start_request(host, slot, cmd);  in dw_mci_start_request()
1220 static void dw_mci_queue_request(struct dw_mci *host, struct dw_mci_slot *slot,  in dw_mci_queue_request()  argument
1224 		 host->state);  in dw_mci_queue_request()
1228 	if (host->state == STATE_WAITING_CMD11_DONE) {  in dw_mci_queue_request()
1236 		host->state = STATE_IDLE;  in dw_mci_queue_request()
1239 	if (host->state == STATE_IDLE) {  in dw_mci_queue_request()
1240 		host->state = STATE_SENDING_CMD;  in dw_mci_queue_request()
1241 		dw_mci_start_request(host, slot);  in dw_mci_queue_request()
1243 		list_add_tail(&slot->queue_node, &host->queue);  in dw_mci_queue_request()
1250 	struct dw_mci *host = slot->host;  in dw_mci_request()  local
1259 	spin_lock_bh(&host->lock);  in dw_mci_request()
1262 		spin_unlock_bh(&host->lock);  in dw_mci_request()
1268 	dw_mci_queue_request(host, slot, mrq);  in dw_mci_request()
1270 	spin_unlock_bh(&host->lock);  in dw_mci_request()
1276 	const struct dw_mci_drv_data *drv_data = slot->host->drv_data;  in dw_mci_set_ios()
1292 	regs = mci_readl(slot->host, UHS_REG);  in dw_mci_set_ios()
1302 	mci_writel(slot->host, UHS_REG, regs);  in dw_mci_set_ios()
1303 	slot->host->timing = ios->timing;  in dw_mci_set_ios()
1312 		drv_data->set_ios(slot->host, ios);  in dw_mci_set_ios()
1320 				dev_err(slot->host->dev,  in dw_mci_set_ios()
1327 		regs = mci_readl(slot->host, PWREN);  in dw_mci_set_ios()
1329 		mci_writel(slot->host, PWREN, regs);  in dw_mci_set_ios()
1332 		if (!slot->host->vqmmc_enabled) {  in dw_mci_set_ios()
1336 					dev_err(slot->host->dev,  in dw_mci_set_ios()
1339 					slot->host->vqmmc_enabled = true;  in dw_mci_set_ios()
1343 				slot->host->vqmmc_enabled = true;  in dw_mci_set_ios()
1347 			dw_mci_ctrl_reset(slot->host,  in dw_mci_set_ios()
1362 		if (!IS_ERR(mmc->supply.vqmmc) && slot->host->vqmmc_enabled)  in dw_mci_set_ios()
1364 		slot->host->vqmmc_enabled = false;  in dw_mci_set_ios()
1366 		regs = mci_readl(slot->host, PWREN);  in dw_mci_set_ios()
1368 		mci_writel(slot->host, PWREN, regs);  in dw_mci_set_ios()
1374 	if (slot->host->state == STATE_WAITING_CMD11_DONE && ios->clock != 0)  in dw_mci_set_ios()
1375 		slot->host->state = STATE_IDLE;  in dw_mci_set_ios()
1387 	status = mci_readl(slot->host, STATUS);  in dw_mci_card_busy()
1395 	struct dw_mci *host = slot->host;  in dw_mci_switch_voltage()  local
1396 	const struct dw_mci_drv_data *drv_data = host->drv_data;  in dw_mci_switch_voltage()
1409 	uhs = mci_readl(host, UHS_REG);  in dw_mci_switch_voltage()
1425 	mci_writel(host, UHS_REG, uhs);  in dw_mci_switch_voltage()
1441 			mci_readl(slot->host, WRTPRT) & (1 << slot->id) ? 1 : 0;  in dw_mci_get_ro()
1453 	struct dw_mci_board *brd = slot->host->pdata;  in dw_mci_get_cd()
1454 	struct dw_mci *host = slot->host;  in dw_mci_get_cd()  local
1464 		present = (mci_readl(slot->host, CDETECT) & (1 << slot->id))  in dw_mci_get_cd()
1467 	spin_lock_bh(&host->lock);  in dw_mci_get_cd()
1475 	spin_unlock_bh(&host->lock);  in dw_mci_get_cd()
1483 	struct dw_mci *host = slot->host;  in dw_mci_init_card()  local
1495 		clk_en_a_old = mci_readl(host, CLKENA);  in dw_mci_init_card()
1507 			mci_writel(host, CLKENA, clk_en_a);  in dw_mci_init_card()
1517 	struct dw_mci *host = slot->host;  in dw_mci_enable_sdio_irq()  local
1521 	spin_lock_irqsave(&host->irq_lock, irqflags);  in dw_mci_enable_sdio_irq()
1524 	int_mask = mci_readl(host, INTMASK);  in dw_mci_enable_sdio_irq()
1529 	mci_writel(host, INTMASK, int_mask);  in dw_mci_enable_sdio_irq()
1531 	spin_unlock_irqrestore(&host->irq_lock, irqflags);  in dw_mci_enable_sdio_irq()
1537 	struct dw_mci *host = slot->host;  in dw_mci_execute_tuning()  local
1538 	const struct dw_mci_drv_data *drv_data = host->drv_data;  in dw_mci_execute_tuning()
1550 	struct dw_mci *host = slot->host;  in dw_mci_prepare_hs400_tuning()  local
1551 	const struct dw_mci_drv_data *drv_data = host->drv_data;  in dw_mci_prepare_hs400_tuning()
1554 		return drv_data->prepare_hs400_tuning(host, ios);  in dw_mci_prepare_hs400_tuning()
1574 static void dw_mci_request_end(struct dw_mci *host, struct mmc_request *mrq)  in dw_mci_request_end()  argument
1575 	__releases(&host->lock)  in dw_mci_request_end()
1576 	__acquires(&host->lock)  in dw_mci_request_end()
1579 	struct mmc_host	*prev_mmc = host->cur_slot->mmc;  in dw_mci_request_end()
1581 	WARN_ON(host->cmd || host->data);  in dw_mci_request_end()
1583 	host->cur_slot->mrq = NULL;  in dw_mci_request_end()
1584 	host->mrq = NULL;  in dw_mci_request_end()
1585 	if (!list_empty(&host->queue)) {  in dw_mci_request_end()
1586 		slot = list_entry(host->queue.next,  in dw_mci_request_end()
1589 		dev_vdbg(host->dev, "list not empty: %s is next\n",  in dw_mci_request_end()
1591 		host->state = STATE_SENDING_CMD;  in dw_mci_request_end()
1592 		dw_mci_start_request(host, slot);  in dw_mci_request_end()
1594 		dev_vdbg(host->dev, "list empty\n");  in dw_mci_request_end()
1596 		if (host->state == STATE_SENDING_CMD11)  in dw_mci_request_end()
1597 			host->state = STATE_WAITING_CMD11_DONE;  in dw_mci_request_end()
1599 			host->state = STATE_IDLE;  in dw_mci_request_end()
1602 	spin_unlock(&host->lock);  in dw_mci_request_end()
1604 	spin_lock(&host->lock);  in dw_mci_request_end()
1607 static int dw_mci_command_complete(struct dw_mci *host, struct mmc_command *cmd)  in dw_mci_command_complete()  argument
1609 	u32 status = host->cmd_status;  in dw_mci_command_complete()
1611 	host->cmd_status = 0;  in dw_mci_command_complete()
1616 			cmd->resp[3] = mci_readl(host, RESP0);  in dw_mci_command_complete()
1617 			cmd->resp[2] = mci_readl(host, RESP1);  in dw_mci_command_complete()
1618 			cmd->resp[1] = mci_readl(host, RESP2);  in dw_mci_command_complete()
1619 			cmd->resp[0] = mci_readl(host, RESP3);  in dw_mci_command_complete()
1621 			cmd->resp[0] = mci_readl(host, RESP0);  in dw_mci_command_complete()
1639 		if (host->quirks & DW_MCI_QUIRK_RETRY_DELAY)  in dw_mci_command_complete()
1646 static int dw_mci_data_complete(struct dw_mci *host, struct mmc_data *data)  in dw_mci_data_complete()  argument
1648 	u32 status = host->data_status;  in dw_mci_data_complete()
1656 			if (host->dir_status ==  in dw_mci_data_complete()
1665 			} else if (host->dir_status ==  in dw_mci_data_complete()
1674 		dev_dbg(host->dev, "data error, status 0x%08x\n", status);  in dw_mci_data_complete()
1680 		dw_mci_reset(host);  in dw_mci_data_complete()
1689 static void dw_mci_set_drto(struct dw_mci *host)  in dw_mci_set_drto()  argument
1694 	drto_clks = mci_readl(host, TMOUT) >> 8;  in dw_mci_set_drto()
1695 	drto_ms = DIV_ROUND_UP(drto_clks, host->bus_hz / 1000);  in dw_mci_set_drto()
1700 	mod_timer(&host->dto_timer, jiffies + msecs_to_jiffies(drto_ms));  in dw_mci_set_drto()
1705 	struct dw_mci *host = (struct dw_mci *)priv;  in dw_mci_tasklet_func()  local
1713 	spin_lock(&host->lock);  in dw_mci_tasklet_func()
1715 	state = host->state;  in dw_mci_tasklet_func()
1716 	data = host->data;  in dw_mci_tasklet_func()
1717 	mrq = host->mrq;  in dw_mci_tasklet_func()
1730 						&host->pending_events))  in dw_mci_tasklet_func()
1733 			cmd = host->cmd;  in dw_mci_tasklet_func()
1734 			host->cmd = NULL;  in dw_mci_tasklet_func()
1735 			set_bit(EVENT_CMD_COMPLETE, &host->completed_events);  in dw_mci_tasklet_func()
1736 			err = dw_mci_command_complete(host, cmd);  in dw_mci_tasklet_func()
1739 				__dw_mci_start_request(host, host->cur_slot,  in dw_mci_tasklet_func()
1745 				dw_mci_stop_dma(host);  in dw_mci_tasklet_func()
1746 				send_stop_abort(host, data);  in dw_mci_tasklet_func()
1752 				dw_mci_request_end(host, mrq);  in dw_mci_tasklet_func()
1769 					       &host->pending_events)) {  in dw_mci_tasklet_func()
1770 				dw_mci_stop_dma(host);  in dw_mci_tasklet_func()
1772 				    !(host->data_status & (SDMMC_INT_DRTO |  in dw_mci_tasklet_func()
1774 					send_stop_abort(host, data);  in dw_mci_tasklet_func()
1780 						&host->pending_events)) {  in dw_mci_tasklet_func()
1785 				if ((host->quirks & DW_MCI_QUIRK_BROKEN_DTO) &&  in dw_mci_tasklet_func()
1786 				    (host->dir_status == DW_MCI_RECV_STATUS))  in dw_mci_tasklet_func()
1787 					dw_mci_set_drto(host);  in dw_mci_tasklet_func()
1791 			set_bit(EVENT_XFER_COMPLETE, &host->completed_events);  in dw_mci_tasklet_func()
1807 					       &host->pending_events)) {  in dw_mci_tasklet_func()
1808 				dw_mci_stop_dma(host);  in dw_mci_tasklet_func()
1810 				    !(host->data_status & (SDMMC_INT_DRTO |  in dw_mci_tasklet_func()
1812 					send_stop_abort(host, data);  in dw_mci_tasklet_func()
1822 						&host->pending_events)) {  in dw_mci_tasklet_func()
1828 				if ((host->quirks & DW_MCI_QUIRK_BROKEN_DTO) &&  in dw_mci_tasklet_func()
1829 				    (host->dir_status == DW_MCI_RECV_STATUS))  in dw_mci_tasklet_func()
1830 					dw_mci_set_drto(host);  in dw_mci_tasklet_func()
1834 			host->data = NULL;  in dw_mci_tasklet_func()
1835 			set_bit(EVENT_DATA_COMPLETE, &host->completed_events);  in dw_mci_tasklet_func()
1836 			err = dw_mci_data_complete(host, data);  in dw_mci_tasklet_func()
1842 					dw_mci_request_end(host, mrq);  in dw_mci_tasklet_func()
1848 					send_stop_abort(host, data);  in dw_mci_tasklet_func()
1860 					      &host->pending_events)) {  in dw_mci_tasklet_func()
1861 					host->cmd = NULL;  in dw_mci_tasklet_func()
1862 					dw_mci_request_end(host, mrq);  in dw_mci_tasklet_func()
1877 						&host->pending_events))  in dw_mci_tasklet_func()
1882 				dw_mci_reset(host);  in dw_mci_tasklet_func()
1884 			host->cmd = NULL;  in dw_mci_tasklet_func()
1885 			host->data = NULL;  in dw_mci_tasklet_func()
1888 				dw_mci_command_complete(host, mrq->stop);  in dw_mci_tasklet_func()
1890 				host->cmd_status = 0;  in dw_mci_tasklet_func()
1892 			dw_mci_request_end(host, mrq);  in dw_mci_tasklet_func()
1897 						&host->pending_events))  in dw_mci_tasklet_func()
1905 	host->state = state;  in dw_mci_tasklet_func()
1907 	spin_unlock(&host->lock);  in dw_mci_tasklet_func()
1912 static void dw_mci_set_part_bytes(struct dw_mci *host, void *buf, int cnt)  in dw_mci_set_part_bytes()  argument
1914 	memcpy((void *)&host->part_buf, buf, cnt);  in dw_mci_set_part_bytes()
1915 	host->part_buf_count = cnt;  in dw_mci_set_part_bytes()
1919 static int dw_mci_push_part_bytes(struct dw_mci *host, void *buf, int cnt)  in dw_mci_push_part_bytes()  argument
1921 	cnt = min(cnt, (1 << host->data_shift) - host->part_buf_count);  in dw_mci_push_part_bytes()
1922 	memcpy((void *)&host->part_buf + host->part_buf_count, buf, cnt);  in dw_mci_push_part_bytes()
1923 	host->part_buf_count += cnt;  in dw_mci_push_part_bytes()
1928 static int dw_mci_pull_part_bytes(struct dw_mci *host, void *buf, int cnt)  in dw_mci_pull_part_bytes()  argument
1930 	cnt = min_t(int, cnt, host->part_buf_count);  in dw_mci_pull_part_bytes()
1932 		memcpy(buf, (void *)&host->part_buf + host->part_buf_start,  in dw_mci_pull_part_bytes()
1934 		host->part_buf_count -= cnt;  in dw_mci_pull_part_bytes()
1935 		host->part_buf_start += cnt;  in dw_mci_pull_part_bytes()
1941 static void dw_mci_pull_final_bytes(struct dw_mci *host, void *buf, int cnt)  in dw_mci_pull_final_bytes()  argument
1943 	memcpy(buf, &host->part_buf, cnt);  in dw_mci_pull_final_bytes()
1944 	host->part_buf_start = cnt;  in dw_mci_pull_final_bytes()
1945 	host->part_buf_count = (1 << host->data_shift) - cnt;  in dw_mci_pull_final_bytes()
1948 static void dw_mci_push_data16(struct dw_mci *host, void *buf, int cnt)  in dw_mci_push_data16()  argument
1950 	struct mmc_data *data = host->data;  in dw_mci_push_data16()
1954 	if (unlikely(host->part_buf_count)) {  in dw_mci_push_data16()
1955 		int len = dw_mci_push_part_bytes(host, buf, cnt);  in dw_mci_push_data16()
1959 		if (host->part_buf_count == 2) {  in dw_mci_push_data16()
1960 			mci_fifo_writew(host->fifo_reg, host->part_buf16);  in dw_mci_push_data16()
1961 			host->part_buf_count = 0;  in dw_mci_push_data16()
1977 				mci_fifo_writew(host->fifo_reg, aligned_buf[i]);  in dw_mci_push_data16()
1985 			mci_fifo_writew(host->fifo_reg, *pdata++);  in dw_mci_push_data16()
1990 		dw_mci_set_part_bytes(host, buf, cnt);  in dw_mci_push_data16()
1994 			mci_fifo_writew(host->fifo_reg, host->part_buf16);  in dw_mci_push_data16()
1998 static void dw_mci_pull_data16(struct dw_mci *host, void *buf, int cnt)  in dw_mci_pull_data16()  argument
2010 				aligned_buf[i] = mci_fifo_readw(host->fifo_reg);  in dw_mci_pull_data16()
2022 			*pdata++ = mci_fifo_readw(host->fifo_reg);  in dw_mci_pull_data16()
2026 		host->part_buf16 = mci_fifo_readw(host->fifo_reg);  in dw_mci_pull_data16()
2027 		dw_mci_pull_final_bytes(host, buf, cnt);  in dw_mci_pull_data16()
2031 static void dw_mci_push_data32(struct dw_mci *host, void *buf, int cnt)  in dw_mci_push_data32()  argument
2033 	struct mmc_data *data = host->data;  in dw_mci_push_data32()
2037 	if (unlikely(host->part_buf_count)) {  in dw_mci_push_data32()
2038 		int len = dw_mci_push_part_bytes(host, buf, cnt);  in dw_mci_push_data32()
2042 		if (host->part_buf_count == 4) {  in dw_mci_push_data32()
2043 			mci_fifo_writel(host->fifo_reg,	host->part_buf32);  in dw_mci_push_data32()
2044 			host->part_buf_count = 0;  in dw_mci_push_data32()
2060 				mci_fifo_writel(host->fifo_reg,	aligned_buf[i]);  in dw_mci_push_data32()
2068 			mci_fifo_writel(host->fifo_reg, *pdata++);  in dw_mci_push_data32()
2073 		dw_mci_set_part_bytes(host, buf, cnt);  in dw_mci_push_data32()
2077 			mci_fifo_writel(host->fifo_reg, host->part_buf32);  in dw_mci_push_data32()
2081 static void dw_mci_pull_data32(struct dw_mci *host, void *buf, int cnt)  in dw_mci_pull_data32()  argument
2093 				aligned_buf[i] = mci_fifo_readl(host->fifo_reg);  in dw_mci_pull_data32()
2105 			*pdata++ = mci_fifo_readl(host->fifo_reg);  in dw_mci_pull_data32()
2109 		host->part_buf32 = mci_fifo_readl(host->fifo_reg);  in dw_mci_pull_data32()
2110 		dw_mci_pull_final_bytes(host, buf, cnt);  in dw_mci_pull_data32()
2114 static void dw_mci_push_data64(struct dw_mci *host, void *buf, int cnt)  in dw_mci_push_data64()  argument
2116 	struct mmc_data *data = host->data;  in dw_mci_push_data64()
2120 	if (unlikely(host->part_buf_count)) {  in dw_mci_push_data64()
2121 		int len = dw_mci_push_part_bytes(host, buf, cnt);  in dw_mci_push_data64()
2126 		if (host->part_buf_count == 8) {  in dw_mci_push_data64()
2127 			mci_fifo_writeq(host->fifo_reg,	host->part_buf);  in dw_mci_push_data64()
2128 			host->part_buf_count = 0;  in dw_mci_push_data64()
2144 				mci_fifo_writeq(host->fifo_reg,	aligned_buf[i]);  in dw_mci_push_data64()
2152 			mci_fifo_writeq(host->fifo_reg, *pdata++);  in dw_mci_push_data64()
2157 		dw_mci_set_part_bytes(host, buf, cnt);  in dw_mci_push_data64()
2161 			mci_fifo_writeq(host->fifo_reg, host->part_buf);  in dw_mci_push_data64()
2165 static void dw_mci_pull_data64(struct dw_mci *host, void *buf, int cnt)  in dw_mci_pull_data64()  argument
2177 				aligned_buf[i] = mci_fifo_readq(host->fifo_reg);  in dw_mci_pull_data64()
2190 			*pdata++ = mci_fifo_readq(host->fifo_reg);  in dw_mci_pull_data64()
2194 		host->part_buf = mci_fifo_readq(host->fifo_reg);  in dw_mci_pull_data64()
2195 		dw_mci_pull_final_bytes(host, buf, cnt);  in dw_mci_pull_data64()
2199 static void dw_mci_pull_data(struct dw_mci *host, void *buf, int cnt)  in dw_mci_pull_data()  argument
2204 	len = dw_mci_pull_part_bytes(host, buf, cnt);  in dw_mci_pull_data()
2211 	host->pull_data(host, buf, cnt);  in dw_mci_pull_data()
2214 static void dw_mci_read_data_pio(struct dw_mci *host, bool dto)  in dw_mci_read_data_pio()  argument
2216 	struct sg_mapping_iter *sg_miter = &host->sg_miter;  in dw_mci_read_data_pio()
2219 	struct mmc_data	*data = host->data;  in dw_mci_read_data_pio()
2220 	int shift = host->data_shift;  in dw_mci_read_data_pio()
2229 		host->sg = sg_miter->piter.sg;  in dw_mci_read_data_pio()
2235 			fcnt = (SDMMC_GET_FCNT(mci_readl(host, STATUS))  in dw_mci_read_data_pio()
2236 					<< shift) + host->part_buf_count;  in dw_mci_read_data_pio()
2240 			dw_mci_pull_data(host, (void *)(buf + offset), len);  in dw_mci_read_data_pio()
2247 		status = mci_readl(host, MINTSTS);  in dw_mci_read_data_pio()
2248 		mci_writel(host, RINTSTS, SDMMC_INT_RXDR);  in dw_mci_read_data_pio()
2251 		 (dto && SDMMC_GET_FCNT(mci_readl(host, STATUS))));  in dw_mci_read_data_pio()
2263 	host->sg = NULL;  in dw_mci_read_data_pio()
2265 	set_bit(EVENT_XFER_COMPLETE, &host->pending_events);  in dw_mci_read_data_pio()
2268 static void dw_mci_write_data_pio(struct dw_mci *host)  in dw_mci_write_data_pio()  argument
2270 	struct sg_mapping_iter *sg_miter = &host->sg_miter;  in dw_mci_write_data_pio()
2273 	struct mmc_data	*data = host->data;  in dw_mci_write_data_pio()
2274 	int shift = host->data_shift;  in dw_mci_write_data_pio()
2277 	unsigned int fifo_depth = host->fifo_depth;  in dw_mci_write_data_pio()
2284 		host->sg = sg_miter->piter.sg;  in dw_mci_write_data_pio()
2291 				 SDMMC_GET_FCNT(mci_readl(host, STATUS)))  in dw_mci_write_data_pio()
2292 					<< shift) - host->part_buf_count;  in dw_mci_write_data_pio()
2296 			host->push_data(host, (void *)(buf + offset), len);  in dw_mci_write_data_pio()
2303 		status = mci_readl(host, MINTSTS);  in dw_mci_write_data_pio()
2304 		mci_writel(host, RINTSTS, SDMMC_INT_TXDR);  in dw_mci_write_data_pio()
2317 	host->sg = NULL;  in dw_mci_write_data_pio()
2319 	set_bit(EVENT_XFER_COMPLETE, &host->pending_events);  in dw_mci_write_data_pio()
2322 static void dw_mci_cmd_interrupt(struct dw_mci *host, u32 status)  in dw_mci_cmd_interrupt()  argument
2324 	if (!host->cmd_status)  in dw_mci_cmd_interrupt()
2325 		host->cmd_status = status;  in dw_mci_cmd_interrupt()
2329 	set_bit(EVENT_CMD_COMPLETE, &host->pending_events);  in dw_mci_cmd_interrupt()
2330 	tasklet_schedule(&host->tasklet);  in dw_mci_cmd_interrupt()
2333 static void dw_mci_handle_cd(struct dw_mci *host)  in dw_mci_handle_cd()  argument
2337 	for (i = 0; i < host->num_slots; i++) {  in dw_mci_handle_cd()
2338 		struct dw_mci_slot *slot = host->slot[i];  in dw_mci_handle_cd()
2346 			msecs_to_jiffies(host->pdata->detect_delay_ms));  in dw_mci_handle_cd()
2352 	struct dw_mci *host = dev_id;  in dw_mci_interrupt()  local
2356 	pending = mci_readl(host, MINTSTS); /* read-only mask reg */  in dw_mci_interrupt()
2362 	if (host->quirks & DW_MCI_QUIRK_IDMAC_DTO) {  in dw_mci_interrupt()
2364 		    ((mci_readl(host, STATUS) >> 17) & 0x1fff))  in dw_mci_interrupt()
2370 		if ((host->state == STATE_SENDING_CMD11) &&  in dw_mci_interrupt()
2374 			mci_writel(host, RINTSTS, SDMMC_INT_VOLT_SWITCH);  in dw_mci_interrupt()
2381 			spin_lock_irqsave(&host->irq_lock, irqflags);  in dw_mci_interrupt()
2382 			dw_mci_cmd_interrupt(host, pending);  in dw_mci_interrupt()
2383 			spin_unlock_irqrestore(&host->irq_lock, irqflags);  in dw_mci_interrupt()
2385 			del_timer(&host->cmd11_timer);  in dw_mci_interrupt()
2389 			mci_writel(host, RINTSTS, DW_MCI_CMD_ERROR_FLAGS);  in dw_mci_interrupt()
2390 			host->cmd_status = pending;  in dw_mci_interrupt()
2392 			set_bit(EVENT_CMD_COMPLETE, &host->pending_events);  in dw_mci_interrupt()
2397 			mci_writel(host, RINTSTS, DW_MCI_DATA_ERROR_FLAGS);  in dw_mci_interrupt()
2398 			host->data_status = pending;  in dw_mci_interrupt()
2400 			set_bit(EVENT_DATA_ERROR, &host->pending_events);  in dw_mci_interrupt()
2401 			tasklet_schedule(&host->tasklet);  in dw_mci_interrupt()
2405 			if (host->quirks & DW_MCI_QUIRK_BROKEN_DTO)  in dw_mci_interrupt()
2406 				del_timer(&host->dto_timer);  in dw_mci_interrupt()
2408 			mci_writel(host, RINTSTS, SDMMC_INT_DATA_OVER);  in dw_mci_interrupt()
2409 			if (!host->data_status)  in dw_mci_interrupt()
2410 				host->data_status = pending;  in dw_mci_interrupt()
2412 			if (host->dir_status == DW_MCI_RECV_STATUS) {  in dw_mci_interrupt()
2413 				if (host->sg != NULL)  in dw_mci_interrupt()
2414 					dw_mci_read_data_pio(host, true);  in dw_mci_interrupt()
2416 			set_bit(EVENT_DATA_COMPLETE, &host->pending_events);  in dw_mci_interrupt()
2417 			tasklet_schedule(&host->tasklet);  in dw_mci_interrupt()
2421 			mci_writel(host, RINTSTS, SDMMC_INT_RXDR);  in dw_mci_interrupt()
2422 			if (host->dir_status == DW_MCI_RECV_STATUS && host->sg)  in dw_mci_interrupt()
2423 				dw_mci_read_data_pio(host, false);  in dw_mci_interrupt()
2427 			mci_writel(host, RINTSTS, SDMMC_INT_TXDR);  in dw_mci_interrupt()
2428 			if (host->dir_status == DW_MCI_SEND_STATUS && host->sg)  in dw_mci_interrupt()
2429 				dw_mci_write_data_pio(host);  in dw_mci_interrupt()
2433 			mci_writel(host, RINTSTS, SDMMC_INT_CMD_DONE);  in dw_mci_interrupt()
2434 			dw_mci_cmd_interrupt(host, pending);  in dw_mci_interrupt()
2438 			mci_writel(host, RINTSTS, SDMMC_INT_CD);  in dw_mci_interrupt()
2439 			dw_mci_handle_cd(host);  in dw_mci_interrupt()
2443 		for (i = 0; i < host->num_slots; i++) {  in dw_mci_interrupt()
2444 			struct dw_mci_slot *slot = host->slot[i];  in dw_mci_interrupt()
2450 				mci_writel(host, RINTSTS,  in dw_mci_interrupt()
2458 	if (host->use_dma != TRANS_MODE_IDMAC)  in dw_mci_interrupt()
2462 	if (host->dma_64bit_address == 1) {  in dw_mci_interrupt()
2463 		pending = mci_readl(host, IDSTS64);  in dw_mci_interrupt()
2465 			mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_TI |  in dw_mci_interrupt()
2467 			mci_writel(host, IDSTS64, SDMMC_IDMAC_INT_NI);  in dw_mci_interrupt()
2468 			host->dma_ops->complete((void *)host);  in dw_mci_interrupt()
2471 		pending = mci_readl(host, IDSTS);  in dw_mci_interrupt()
2473 			mci_writel(host, IDSTS, SDMMC_IDMAC_INT_TI |  in dw_mci_interrupt()
2475 			mci_writel(host, IDSTS, SDMMC_IDMAC_INT_NI);  in dw_mci_interrupt()
2476 			host->dma_ops->complete((void *)host);  in dw_mci_interrupt()
2524 static int dw_mci_init_slot(struct dw_mci *host, unsigned int id)  in dw_mci_init_slot()  argument
2528 	const struct dw_mci_drv_data *drv_data = host->drv_data;  in dw_mci_init_slot()
2532 	mmc = mmc_alloc_host(sizeof(struct dw_mci_slot), host->dev);  in dw_mci_init_slot()
2538 	slot->sdio_id = host->sdio_id0 + id;  in dw_mci_init_slot()
2540 	slot->host = host;  in dw_mci_init_slot()
2541 	host->slot[id] = slot;  in dw_mci_init_slot()
2544 	if (of_property_read_u32_array(host->dev->of_node,  in dw_mci_init_slot()
2561 	if (host->pdata->caps)  in dw_mci_init_slot()
2562 		mmc->caps = host->pdata->caps;  in dw_mci_init_slot()
2564 	if (host->pdata->pm_caps)  in dw_mci_init_slot()
2565 		mmc->pm_caps = host->pdata->pm_caps;  in dw_mci_init_slot()
2567 	if (host->dev->of_node) {  in dw_mci_init_slot()
2568 		ctrl_id = of_alias_get_id(host->dev->of_node, "mshc");  in dw_mci_init_slot()
2572 		ctrl_id = to_platform_device(host->dev)->id;  in dw_mci_init_slot()
2577 	if (host->pdata->caps2)  in dw_mci_init_slot()
2578 		mmc->caps2 = host->pdata->caps2;  in dw_mci_init_slot()
2587 	if (host->use_dma == TRANS_MODE_IDMAC) {  in dw_mci_init_slot()
2588 		mmc->max_segs = host->ring_size;  in dw_mci_init_slot()
2591 		mmc->max_req_size = mmc->max_seg_size * host->ring_size;  in dw_mci_init_slot()
2593 	} else if (host->use_dma == TRANS_MODE_EDMAC) {  in dw_mci_init_slot()
2634 	slot->host->slot[id] = NULL;  in dw_mci_cleanup_slot()
2638 static void dw_mci_init_dma(struct dw_mci *host)  in dw_mci_init_dma()  argument
2641 	struct device *dev = host->dev;  in dw_mci_init_dma()
2655 	host->use_dma = SDMMC_GET_TRANS_MODE(mci_readl(host, HCON));  in dw_mci_init_dma()
2656 	if (host->use_dma == DMA_INTERFACE_IDMA) {  in dw_mci_init_dma()
2657 		host->use_dma = TRANS_MODE_IDMAC;  in dw_mci_init_dma()
2658 	} else if (host->use_dma == DMA_INTERFACE_DWDMA ||  in dw_mci_init_dma()
2659 		   host->use_dma == DMA_INTERFACE_GDMA) {  in dw_mci_init_dma()
2660 		host->use_dma = TRANS_MODE_EDMAC;  in dw_mci_init_dma()
2666 	if (host->use_dma == TRANS_MODE_IDMAC) {  in dw_mci_init_dma()
2671 		addr_config = SDMMC_GET_ADDR_CONFIG(mci_readl(host, HCON));  in dw_mci_init_dma()
2675 			host->dma_64bit_address = 1;  in dw_mci_init_dma()
2676 			dev_info(host->dev,  in dw_mci_init_dma()
2678 			if (!dma_set_mask(host->dev, DMA_BIT_MASK(64)))  in dw_mci_init_dma()
2679 				dma_set_coherent_mask(host->dev,  in dw_mci_init_dma()
2683 			host->dma_64bit_address = 0;  in dw_mci_init_dma()
2684 			dev_info(host->dev,  in dw_mci_init_dma()
2689 		host->sg_cpu = dmam_alloc_coherent(host->dev, PAGE_SIZE,  in dw_mci_init_dma()
2690 						   &host->sg_dma, GFP_KERNEL);  in dw_mci_init_dma()
2691 		if (!host->sg_cpu) {  in dw_mci_init_dma()
2692 			dev_err(host->dev,  in dw_mci_init_dma()
2698 		host->dma_ops = &dw_mci_idmac_ops;  in dw_mci_init_dma()
2699 		dev_info(host->dev, "Using internal DMA controller.\n");  in dw_mci_init_dma()
2706 		host->dma_ops = &dw_mci_edmac_ops;  in dw_mci_init_dma()
2707 		dev_info(host->dev, "Using external DMA controller.\n");  in dw_mci_init_dma()
2710 	if (host->dma_ops->init && host->dma_ops->start &&  in dw_mci_init_dma()
2711 	    host->dma_ops->stop && host->dma_ops->cleanup) {  in dw_mci_init_dma()
2712 		if (host->dma_ops->init(host)) {  in dw_mci_init_dma()
2713 			dev_err(host->dev, "%s: Unable to initialize DMA Controller.\n",  in dw_mci_init_dma()
2718 		dev_err(host->dev, "DMA initialization not found.\n");  in dw_mci_init_dma()
2725 	dev_info(host->dev, "Using PIO mode.\n");  in dw_mci_init_dma()
2726 	host->use_dma = TRANS_MODE_PIO;  in dw_mci_init_dma()
2729 static bool dw_mci_ctrl_reset(struct dw_mci *host, u32 reset)  in dw_mci_ctrl_reset()  argument
2734 	ctrl = mci_readl(host, CTRL);  in dw_mci_ctrl_reset()
2736 	mci_writel(host, CTRL, ctrl);  in dw_mci_ctrl_reset()
2740 		ctrl = mci_readl(host, CTRL);  in dw_mci_ctrl_reset()
2745 	dev_err(host->dev,  in dw_mci_ctrl_reset()
2752 static bool dw_mci_reset(struct dw_mci *host)  in dw_mci_reset()  argument
2761 	if (host->sg) {  in dw_mci_reset()
2762 		sg_miter_stop(&host->sg_miter);  in dw_mci_reset()
2763 		host->sg = NULL;  in dw_mci_reset()
2766 	if (host->use_dma)  in dw_mci_reset()
2769 	if (dw_mci_ctrl_reset(host, flags)) {  in dw_mci_reset()
2774 		mci_writel(host, RINTSTS, 0xFFFFFFFF);  in dw_mci_reset()
2777 		if (host->use_dma) {  in dw_mci_reset()
2782 				status = mci_readl(host, STATUS);  in dw_mci_reset()
2789 				dev_err(host->dev,  in dw_mci_reset()
2796 			if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_FIFO_RESET))  in dw_mci_reset()
2801 		if (!(mci_readl(host, CTRL) & SDMMC_CTRL_RESET)) {  in dw_mci_reset()
2802 			dev_err(host->dev,  in dw_mci_reset()
2809 	if (host->use_dma == TRANS_MODE_IDMAC)  in dw_mci_reset()
2811 		dw_mci_idmac_reset(host);  in dw_mci_reset()
2817 	mci_send_cmd(host->cur_slot, SDMMC_CMD_UPD_CLK, 0);  in dw_mci_reset()
2824 	struct dw_mci *host = (struct dw_mci *)arg;  in dw_mci_cmd11_timer()  local
2826 	if (host->state != STATE_SENDING_CMD11) {  in dw_mci_cmd11_timer()
2827 		dev_warn(host->dev, "Unexpected CMD11 timeout\n");  in dw_mci_cmd11_timer()
2831 	host->cmd_status = SDMMC_INT_RTO;  in dw_mci_cmd11_timer()
2832 	set_bit(EVENT_CMD_COMPLETE, &host->pending_events);  in dw_mci_cmd11_timer()
2833 	tasklet_schedule(&host->tasklet);  in dw_mci_cmd11_timer()
2838 	struct dw_mci *host = (struct dw_mci *)arg;  in dw_mci_dto_timer()  local
2840 	switch (host->state) {  in dw_mci_dto_timer()
2848 		host->data_status = SDMMC_INT_DRTO;  in dw_mci_dto_timer()
2849 		set_bit(EVENT_DATA_ERROR, &host->pending_events);  in dw_mci_dto_timer()
2850 		set_bit(EVENT_DATA_COMPLETE, &host->pending_events);  in dw_mci_dto_timer()
2851 		tasklet_schedule(&host->tasklet);  in dw_mci_dto_timer()
2869 static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)  in dw_mci_parse_dt()  argument
2872 	struct device *dev = host->dev;  in dw_mci_parse_dt()
2874 	const struct dw_mci_drv_data *drv_data = host->drv_data;  in dw_mci_parse_dt()
2905 		ret = drv_data->parse_dt(host);  in dw_mci_parse_dt()
2919 static struct dw_mci_board *dw_mci_parse_dt(struct dw_mci *host)  in dw_mci_parse_dt()  argument
2925 static void dw_mci_enable_cd(struct dw_mci *host)  in dw_mci_enable_cd()  argument
2927 	struct dw_mci_board *brd = host->pdata;  in dw_mci_enable_cd()
2937 	for (i = 0; i < host->num_slots; i++) {  in dw_mci_enable_cd()
2938 		struct dw_mci_slot *slot = host->slot[i];  in dw_mci_enable_cd()
2943 	if (i == host->num_slots)  in dw_mci_enable_cd()
2946 	spin_lock_irqsave(&host->irq_lock, irqflags);  in dw_mci_enable_cd()
2947 	temp = mci_readl(host, INTMASK);  in dw_mci_enable_cd()
2949 	mci_writel(host, INTMASK, temp);  in dw_mci_enable_cd()
2950 	spin_unlock_irqrestore(&host->irq_lock, irqflags);  in dw_mci_enable_cd()
2953 int dw_mci_probe(struct dw_mci *host)  in dw_mci_probe()  argument
2955 	const struct dw_mci_drv_data *drv_data = host->drv_data;  in dw_mci_probe()
2960 	if (!host->pdata) {  in dw_mci_probe()
2961 		host->pdata = dw_mci_parse_dt(host);  in dw_mci_probe()
2962 		if (IS_ERR(host->pdata)) {  in dw_mci_probe()
2963 			dev_err(host->dev, "platform data not available\n");  in dw_mci_probe()
2968 	if (host->pdata->num_slots < 1) {  in dw_mci_probe()
2969 		dev_err(host->dev,  in dw_mci_probe()
2974 	host->biu_clk = devm_clk_get(host->dev, "biu");  in dw_mci_probe()
2975 	if (IS_ERR(host->biu_clk)) {  in dw_mci_probe()
2976 		dev_dbg(host->dev, "biu clock not available\n");  in dw_mci_probe()
2978 		ret = clk_prepare_enable(host->biu_clk);  in dw_mci_probe()
2980 			dev_err(host->dev, "failed to enable biu clock\n");  in dw_mci_probe()
2985 	host->ciu_clk = devm_clk_get(host->dev, "ciu");  in dw_mci_probe()
2986 	if (IS_ERR(host->ciu_clk)) {  in dw_mci_probe()
2987 		dev_dbg(host->dev, "ciu clock not available\n");  in dw_mci_probe()
2988 		host->bus_hz = host->pdata->bus_hz;  in dw_mci_probe()
2990 		ret = clk_prepare_enable(host->ciu_clk);  in dw_mci_probe()
2992 			dev_err(host->dev, "failed to enable ciu clock\n");  in dw_mci_probe()
2996 		if (host->pdata->bus_hz) {  in dw_mci_probe()
2997 			ret = clk_set_rate(host->ciu_clk, host->pdata->bus_hz);  in dw_mci_probe()
2999 				dev_warn(host->dev,  in dw_mci_probe()
3001 					 host->pdata->bus_hz);  in dw_mci_probe()
3003 		host->bus_hz = clk_get_rate(host->ciu_clk);  in dw_mci_probe()
3006 	if (!host->bus_hz) {  in dw_mci_probe()
3007 		dev_err(host->dev,  in dw_mci_probe()
3014 		ret = drv_data->init(host);  in dw_mci_probe()
3016 			dev_err(host->dev,  in dw_mci_probe()
3023 		ret = drv_data->setup_clock(host);  in dw_mci_probe()
3025 			dev_err(host->dev,  in dw_mci_probe()
3031 	setup_timer(&host->cmd11_timer,  in dw_mci_probe()
3032 		    dw_mci_cmd11_timer, (unsigned long)host);  in dw_mci_probe()
3034 	host->quirks = host->pdata->quirks;  in dw_mci_probe()
3036 	if (host->quirks & DW_MCI_QUIRK_BROKEN_DTO)  in dw_mci_probe()
3037 		setup_timer(&host->dto_timer,  in dw_mci_probe()
3038 			    dw_mci_dto_timer, (unsigned long)host);  in dw_mci_probe()
3040 	spin_lock_init(&host->lock);  in dw_mci_probe()
3041 	spin_lock_init(&host->irq_lock);  in dw_mci_probe()
3042 	INIT_LIST_HEAD(&host->queue);  in dw_mci_probe()
3048 	i = SDMMC_GET_HDATA_WIDTH(mci_readl(host, HCON));  in dw_mci_probe()
3050 		host->push_data = dw_mci_push_data16;  in dw_mci_probe()
3051 		host->pull_data = dw_mci_pull_data16;  in dw_mci_probe()
3053 		host->data_shift = 1;  in dw_mci_probe()
3055 		host->push_data = dw_mci_push_data64;  in dw_mci_probe()
3056 		host->pull_data = dw_mci_pull_data64;  in dw_mci_probe()
3058 		host->data_shift = 3;  in dw_mci_probe()
3064 		host->push_data = dw_mci_push_data32;  in dw_mci_probe()
3065 		host->pull_data = dw_mci_pull_data32;  in dw_mci_probe()
3067 		host->data_shift = 2;  in dw_mci_probe()
3071 	if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS))  in dw_mci_probe()
3074 	host->dma_ops = host->pdata->dma_ops;  in dw_mci_probe()
3075 	dw_mci_init_dma(host);  in dw_mci_probe()
3078 	mci_writel(host, RINTSTS, 0xFFFFFFFF);  in dw_mci_probe()
3079 	mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */  in dw_mci_probe()
3082 	mci_writel(host, TMOUT, 0xFFFFFFFF);  in dw_mci_probe()
3088 	if (!host->pdata->fifo_depth) {  in dw_mci_probe()
3095 		fifo_size = mci_readl(host, FIFOTH);  in dw_mci_probe()
3098 		fifo_size = host->pdata->fifo_depth;  in dw_mci_probe()
3100 	host->fifo_depth = fifo_size;  in dw_mci_probe()
3101 	host->fifoth_val =  in dw_mci_probe()
3103 	mci_writel(host, FIFOTH, host->fifoth_val);  in dw_mci_probe()
3106 	mci_writel(host, CLKENA, 0);  in dw_mci_probe()
3107 	mci_writel(host, CLKSRC, 0);  in dw_mci_probe()
3113 	host->verid = SDMMC_GET_VERID(mci_readl(host, VERID));  in dw_mci_probe()
3114 	dev_info(host->dev, "Version ID is %04x\n", host->verid);  in dw_mci_probe()
3116 	if (host->verid < DW_MMC_240A)  in dw_mci_probe()
3117 		host->fifo_reg = host->regs + DATA_OFFSET;  in dw_mci_probe()
3119 		host->fifo_reg = host->regs + DATA_240A_OFFSET;  in dw_mci_probe()
3121 	tasklet_init(&host->tasklet, dw_mci_tasklet_func, (unsigned long)host);  in dw_mci_probe()
3122 	ret = devm_request_irq(host->dev, host->irq, dw_mci_interrupt,  in dw_mci_probe()
3123 			       host->irq_flags, "dw-mci", host);  in dw_mci_probe()
3127 	if (host->pdata->num_slots)  in dw_mci_probe()
3128 		host->num_slots = host->pdata->num_slots;  in dw_mci_probe()
3130 		host->num_slots = SDMMC_GET_SLOT_NUM(mci_readl(host, HCON));  in dw_mci_probe()
3136 	mci_writel(host, RINTSTS, 0xFFFFFFFF);  in dw_mci_probe()
3137 	mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER |  in dw_mci_probe()
3141 	mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE);  in dw_mci_probe()
3143 	dev_info(host->dev,  in dw_mci_probe()
3145 		 host->irq, width, fifo_size);  in dw_mci_probe()
3148 	for (i = 0; i < host->num_slots; i++) {  in dw_mci_probe()
3149 		ret = dw_mci_init_slot(host, i);  in dw_mci_probe()
3151 			dev_dbg(host->dev, "slot %d init failed\n", i);  in dw_mci_probe()
3157 		dev_info(host->dev, "%d slots initialized\n", init_slots);  in dw_mci_probe()
3159 		dev_dbg(host->dev,  in dw_mci_probe()
3161 			host->num_slots);  in dw_mci_probe()
3166 	dw_mci_enable_cd(host);  in dw_mci_probe()
3168 	if (host->quirks & DW_MCI_QUIRK_IDMAC_DTO)  in dw_mci_probe()
3169 		dev_info(host->dev, "Internal DMAC interrupt fix enabled.\n");  in dw_mci_probe()
3174 	if (host->use_dma && host->dma_ops->exit)  in dw_mci_probe()
3175 		host->dma_ops->exit(host);  in dw_mci_probe()
3178 	if (!IS_ERR(host->ciu_clk))  in dw_mci_probe()
3179 		clk_disable_unprepare(host->ciu_clk);  in dw_mci_probe()
3182 	if (!IS_ERR(host->biu_clk))  in dw_mci_probe()
3183 		clk_disable_unprepare(host->biu_clk);  in dw_mci_probe()
3189 void dw_mci_remove(struct dw_mci *host)  in dw_mci_remove()  argument
3193 	for (i = 0; i < host->num_slots; i++) {  in dw_mci_remove()
3194 		dev_dbg(host->dev, "remove slot %d\n", i);  in dw_mci_remove()
3195 		if (host->slot[i])  in dw_mci_remove()
3196 			dw_mci_cleanup_slot(host->slot[i], i);  in dw_mci_remove()
3199 	mci_writel(host, RINTSTS, 0xFFFFFFFF);  in dw_mci_remove()
3200 	mci_writel(host, INTMASK, 0); /* disable all mmc interrupt first */  in dw_mci_remove()
3203 	mci_writel(host, CLKENA, 0);  in dw_mci_remove()
3204 	mci_writel(host, CLKSRC, 0);  in dw_mci_remove()
3206 	if (host->use_dma && host->dma_ops->exit)  in dw_mci_remove()
3207 		host->dma_ops->exit(host);  in dw_mci_remove()
3209 	if (!IS_ERR(host->ciu_clk))  in dw_mci_remove()
3210 		clk_disable_unprepare(host->ciu_clk);  in dw_mci_remove()
3212 	if (!IS_ERR(host->biu_clk))  in dw_mci_remove()
3213 		clk_disable_unprepare(host->biu_clk);  in dw_mci_remove()
3223 int dw_mci_suspend(struct dw_mci *host)  in dw_mci_suspend()  argument
3225 	if (host->use_dma && host->dma_ops->exit)  in dw_mci_suspend()
3226 		host->dma_ops->exit(host);  in dw_mci_suspend()
3232 int dw_mci_resume(struct dw_mci *host)  in dw_mci_resume()  argument
3236 	if (!dw_mci_ctrl_reset(host, SDMMC_CTRL_ALL_RESET_FLAGS)) {  in dw_mci_resume()
3241 	if (host->use_dma && host->dma_ops->init)  in dw_mci_resume()
3242 		host->dma_ops->init(host);  in dw_mci_resume()
3248 	mci_writel(host, FIFOTH, host->fifoth_val);  in dw_mci_resume()
3249 	host->prev_blksz = 0;  in dw_mci_resume()
3252 	mci_writel(host, TMOUT, 0xFFFFFFFF);  in dw_mci_resume()
3254 	mci_writel(host, RINTSTS, 0xFFFFFFFF);  in dw_mci_resume()
3255 	mci_writel(host, INTMASK, SDMMC_INT_CMD_DONE | SDMMC_INT_DATA_OVER |  in dw_mci_resume()
3258 	mci_writel(host, CTRL, SDMMC_CTRL_INT_ENABLE);  in dw_mci_resume()
3260 	for (i = 0; i < host->num_slots; i++) {  in dw_mci_resume()
3261 		struct dw_mci_slot *slot = host->slot[i];  in dw_mci_resume()
3272 	dw_mci_enable_cd(host);  in dw_mci_resume()