488 static void emc_ccfifo_writel(struct tegra_emc *emc, u32 value,  in emc_ccfifo_writel()  argument
491 	writel(value, emc->regs + EMC_CCFIFO_DATA);  in emc_ccfifo_writel()
492 	writel(offset, emc->regs + EMC_CCFIFO_ADDR);  in emc_ccfifo_writel()
495 static void emc_seq_update_timing(struct tegra_emc *emc)  in emc_seq_update_timing()  argument
500 	writel(1, emc->regs + EMC_TIMING_CONTROL);  in emc_seq_update_timing()
503 		value = readl(emc->regs + EMC_STATUS);  in emc_seq_update_timing()
509 	dev_err(emc->dev, "timing update timed out\n");  in emc_seq_update_timing()
512 static void emc_seq_disable_auto_cal(struct tegra_emc *emc)  in emc_seq_disable_auto_cal()  argument
517 	writel(0, emc->regs + EMC_AUTO_CAL_INTERVAL);  in emc_seq_disable_auto_cal()
520 		value = readl(emc->regs + EMC_AUTO_CAL_STATUS);  in emc_seq_disable_auto_cal()
526 	dev_err(emc->dev, "auto cal disable timed out\n");  in emc_seq_disable_auto_cal()
529 static void emc_seq_wait_clkchange(struct tegra_emc *emc)  in emc_seq_wait_clkchange()  argument
535 		value = readl(emc->regs + EMC_INTSTATUS);  in emc_seq_wait_clkchange()
541 	dev_err(emc->dev, "clock change timed out\n");  in emc_seq_wait_clkchange()
544 static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc,  in tegra_emc_find_timing()  argument
550 	for (i = 0; i < emc->num_timings; i++) {  in tegra_emc_find_timing()
551 		if (emc->timings[i].rate == rate) {  in tegra_emc_find_timing()
552 			timing = &emc->timings[i];  in tegra_emc_find_timing()
558 		dev_err(emc->dev, "no timing for rate %lu\n", rate);  in tegra_emc_find_timing()
565 int tegra_emc_prepare_timing_change(struct tegra_emc *emc,  in tegra_emc_prepare_timing_change()  argument
568 	struct emc_timing *timing = tegra_emc_find_timing(emc, rate);  in tegra_emc_prepare_timing_change()
569 	struct emc_timing *last = &emc->last_timing;  in tegra_emc_prepare_timing_change()
587 	writel(EMC_INTSTATUS_CLKCHANGE_COMPLETE, emc->regs + EMC_INTSTATUS);  in tegra_emc_prepare_timing_change()
590 	val = readl(emc->regs + EMC_CFG);  in tegra_emc_prepare_timing_change()
593 		writel(val, emc->regs + EMC_CFG);  in tegra_emc_prepare_timing_change()
599 	if (emc->dram_type == DRAM_TYPE_DDR3)  in tegra_emc_prepare_timing_change()
604 	val = readl(emc->regs + EMC_SEL_DPD_CTRL);  in tegra_emc_prepare_timing_change()
607 		writel(val, emc->regs + EMC_SEL_DPD_CTRL);  in tegra_emc_prepare_timing_change()
611 	val = readl(emc->regs + EMC_BGBIAS_CTL0);  in tegra_emc_prepare_timing_change()
626 		writel(val2, emc->regs + EMC_BGBIAS_CTL0);  in tegra_emc_prepare_timing_change()
632 	val = readl(emc->regs + EMC_XM2DQSPADCTRL2);  in tegra_emc_prepare_timing_change()
646 		writel(val, emc->regs + EMC_XM2DQSPADCTRL2);  in tegra_emc_prepare_timing_change()
653 		emc_seq_update_timing(emc);  in tegra_emc_prepare_timing_change()
659 		emc_seq_disable_auto_cal(emc);  in tegra_emc_prepare_timing_change()
661 		       emc->regs + EMC_CTT_TERM_CTRL);  in tegra_emc_prepare_timing_change()
662 		emc_seq_update_timing(emc);  in tegra_emc_prepare_timing_change()
668 		       emc->regs + emc_burst_regs[i]);  in tegra_emc_prepare_timing_change()
670 	writel(timing->emc_xm2dqspadctrl2, emc->regs + EMC_XM2DQSPADCTRL2);  in tegra_emc_prepare_timing_change()
671 	writel(timing->emc_zcal_interval, emc->regs + EMC_ZCAL_INTERVAL);  in tegra_emc_prepare_timing_change()
673 	tegra_mc_write_emem_configuration(emc->mc, timing->rate);  in tegra_emc_prepare_timing_change()
676 	emc_ccfifo_writel(emc, val, EMC_CFG);  in tegra_emc_prepare_timing_change()
680 		emc_ccfifo_writel(emc, timing->emc_auto_cal_config2,  in tegra_emc_prepare_timing_change()
684 		emc_ccfifo_writel(emc, timing->emc_auto_cal_config3,  in tegra_emc_prepare_timing_change()
690 		emc_ccfifo_writel(emc, val, EMC_AUTO_CAL_CONFIG);  in tegra_emc_prepare_timing_change()
694 	if (emc->dram_type == DRAM_TYPE_DDR3 &&  in tegra_emc_prepare_timing_change()
700 			cnt -= emc->dram_num * 256;  in tegra_emc_prepare_timing_change()
713 		writel(val, emc->regs + EMC_MRS_WAIT_CNT);  in tegra_emc_prepare_timing_change()
718 	emc_ccfifo_writel(emc, val, EMC_CFG_2);  in tegra_emc_prepare_timing_change()
721 	if (emc->dram_type == DRAM_TYPE_DDR3 && dll_change == DLL_CHANGE_OFF)  in tegra_emc_prepare_timing_change()
722 		emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_EMRS);  in tegra_emc_prepare_timing_change()
725 	emc_ccfifo_writel(emc, EMC_REFCTRL_DEV_SEL(emc->dram_num),  in tegra_emc_prepare_timing_change()
727 	if (emc->dram_type == DRAM_TYPE_DDR3)  in tegra_emc_prepare_timing_change()
728 		emc_ccfifo_writel(emc, EMC_DRAM_DEV_SEL(emc->dram_num) |  in tegra_emc_prepare_timing_change()
733 	emc_ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE);  in tegra_emc_prepare_timing_change()
736 	if (emc->dram_type == DRAM_TYPE_DDR3)  in tegra_emc_prepare_timing_change()
737 		emc_ccfifo_writel(emc, EMC_DRAM_DEV_SEL(emc->dram_num),  in tegra_emc_prepare_timing_change()
739 	emc_ccfifo_writel(emc, EMC_REFCTRL_DEV_SEL(emc->dram_num) |  in tegra_emc_prepare_timing_change()
744 	if (emc->dram_type == DRAM_TYPE_DDR3) {  in tegra_emc_prepare_timing_change()
746 			emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_EMRS);  in tegra_emc_prepare_timing_change()
748 			emc_ccfifo_writel(emc, timing->emc_mode_2, EMC_EMRS2);  in tegra_emc_prepare_timing_change()
759 			emc_ccfifo_writel(emc, val, EMC_MRS);  in tegra_emc_prepare_timing_change()
763 			emc_ccfifo_writel(emc, timing->emc_mode_2, EMC_MRW2);  in tegra_emc_prepare_timing_change()
765 			emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_MRW);  in tegra_emc_prepare_timing_change()
767 			emc_ccfifo_writel(emc, timing->emc_mode_4, EMC_MRW4);  in tegra_emc_prepare_timing_change()
772 		emc_ccfifo_writel(emc, EMC_ZQ_CAL_LONG_CMD_DEV0, EMC_ZQ_CAL);  in tegra_emc_prepare_timing_change()
773 		if (emc->dram_num > 1)  in tegra_emc_prepare_timing_change()
774 			emc_ccfifo_writel(emc, EMC_ZQ_CAL_LONG_CMD_DEV1,  in tegra_emc_prepare_timing_change()
779 	emc_ccfifo_writel(emc, 0, EMC_CCFIFO_STATUS);  in tegra_emc_prepare_timing_change()
782 		emc_ccfifo_writel(emc, timing->emc_cfg_2, EMC_CFG_2);  in tegra_emc_prepare_timing_change()
785 	emc_seq_disable_auto_cal(emc);  in tegra_emc_prepare_timing_change()
788 	readl(emc->regs + EMC_INTSTATUS);  in tegra_emc_prepare_timing_change()
793 void tegra_emc_complete_timing_change(struct tegra_emc *emc,  in tegra_emc_complete_timing_change()  argument
796 	struct emc_timing *timing = tegra_emc_find_timing(emc, rate);  in tegra_emc_complete_timing_change()
797 	struct emc_timing *last = &emc->last_timing;  in tegra_emc_complete_timing_change()
804 	emc_seq_wait_clkchange(emc);  in tegra_emc_complete_timing_change()
809 		       emc->regs + EMC_AUTO_CAL_INTERVAL);  in tegra_emc_complete_timing_change()
813 		writel(timing->emc_cfg, emc->regs + EMC_CFG);  in tegra_emc_complete_timing_change()
816 	writel(timing->emc_zcal_cnt_long, emc->regs + EMC_ZCAL_WAIT_CNT);  in tegra_emc_complete_timing_change()
819 	if (emc->dram_type == DRAM_TYPE_LPDDR3 &&  in tegra_emc_complete_timing_change()
825 		writel(val, emc->regs + EMC_BGBIAS_CTL0);  in tegra_emc_complete_timing_change()
827 		if (emc->dram_type == DRAM_TYPE_DDR3 &&  in tegra_emc_complete_timing_change()
828 		    readl(emc->regs + EMC_BGBIAS_CTL0) !=  in tegra_emc_complete_timing_change()
831 			       emc->regs + EMC_BGBIAS_CTL0);  in tegra_emc_complete_timing_change()
835 		       emc->regs + EMC_AUTO_CAL_INTERVAL);  in tegra_emc_complete_timing_change()
842 	writel(timing->emc_sel_dpd_ctrl, emc->regs + EMC_SEL_DPD_CTRL);  in tegra_emc_complete_timing_change()
843 	emc_seq_update_timing(emc);  in tegra_emc_complete_timing_change()
845 	emc->last_timing = *timing;  in tegra_emc_complete_timing_change()
850 static void emc_read_current_timing(struct tegra_emc *emc,  in emc_read_current_timing()  argument
857 			readl(emc->regs + emc_burst_regs[i]);  in emc_read_current_timing()
859 	timing->emc_cfg = readl(emc->regs + EMC_CFG);  in emc_read_current_timing()
869 static int emc_init(struct tegra_emc *emc)  in emc_init()  argument
871 	emc->dram_type = readl(emc->regs + EMC_FBIO_CFG5);  in emc_init()
872 	emc->dram_type &= EMC_FBIO_CFG5_DRAM_TYPE_MASK;  in emc_init()
873 	emc->dram_type >>= EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;  in emc_init()
875 	emc->dram_num = tegra_mc_get_emem_device_count(emc->mc);  in emc_init()
877 	emc_read_current_timing(emc, &emc->last_timing);  in emc_init()
882 static int load_one_timing_from_dt(struct tegra_emc *emc,  in load_one_timing_from_dt()  argument
891 		dev_err(emc->dev, "timing %s: failed to read rate: %d\n",  in load_one_timing_from_dt()
902 		dev_err(emc->dev,  in load_one_timing_from_dt()
911 		dev_err(emc->dev, "timing %s: failed to read " #prop ": %d\n", \  in load_one_timing_from_dt()
953 static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc,  in tegra_emc_load_timings_from_dt()  argument
962 	emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing),  in tegra_emc_load_timings_from_dt()
964 	if (!emc->timings)  in tegra_emc_load_timings_from_dt()
967 	emc->num_timings = child_count;  in tegra_emc_load_timings_from_dt()
970 		timing = &emc->timings[i++];  in tegra_emc_load_timings_from_dt()
972 		err = load_one_timing_from_dt(emc, timing, child);  in tegra_emc_load_timings_from_dt()
977 	sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings,  in tegra_emc_load_timings_from_dt()
1032 	struct tegra_emc *emc = s->private;  in emc_debug_supported_rates_show()  local
1036 	for (i = 0; i < emc->num_timings; i++) {  in emc_debug_supported_rates_show()
1037 		struct emc_timing *timing = &emc->timings[i];  in emc_debug_supported_rates_show()
1063 static void emc_debugfs_init(struct device *dev, struct tegra_emc *emc)  in emc_debugfs_init()  argument
1085 	file = debugfs_create_file("supported_rates", S_IRUGO, root, emc,  in emc_debugfs_init()
1095 	struct tegra_emc *emc;  in tegra_emc_probe()  local
1100 	emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);  in tegra_emc_probe()
1101 	if (!emc)  in tegra_emc_probe()
1104 	emc->dev = &pdev->dev;  in tegra_emc_probe()
1107 	emc->regs = devm_ioremap_resource(&pdev->dev, res);  in tegra_emc_probe()
1108 	if (IS_ERR(emc->regs))  in tegra_emc_probe()
1109 		return PTR_ERR(emc->regs);  in tegra_emc_probe()
1123 	emc->mc = platform_get_drvdata(mc);  in tegra_emc_probe()
1124 	if (!emc->mc)  in tegra_emc_probe()
1137 	err = tegra_emc_load_timings_from_dt(emc, np);  in tegra_emc_probe()
1144 	if (emc->num_timings == 0) {  in tegra_emc_probe()
1151 	err = emc_init(emc);  in tegra_emc_probe()
1157 	platform_set_drvdata(pdev, emc);  in tegra_emc_probe()
1160 		emc_debugfs_init(&pdev->dev, emc);  in tegra_emc_probe()