49 static int ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba *hba,
52 static void ufs_qcom_dump_regs(struct ufs_hba *hba, int offset, int len,  in ufs_qcom_dump_regs()  argument
57 			16, 4, (void __force *)hba->mmio_base + offset,  in ufs_qcom_dump_regs()
61 static int ufs_qcom_get_connected_tx_lanes(struct ufs_hba *hba, u32 *tx_lanes)  in ufs_qcom_get_connected_tx_lanes()  argument
65 	err = ufshcd_dme_get(hba,  in ufs_qcom_get_connected_tx_lanes()
68 		dev_err(hba->dev, "%s: couldn't read PA_CONNECTEDTXDATALANES %d\n",  in ufs_qcom_get_connected_tx_lanes()
120 	struct device *dev = host->hba->dev;  in ufs_qcom_enable_lane_clks()
161 	struct device *dev = host->hba->dev;  in ufs_qcom_init_lane_clks()
185 static int ufs_qcom_link_startup_post_change(struct ufs_hba *hba)  in ufs_qcom_link_startup_post_change()  argument
187 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_link_startup_post_change()
192 	err = ufs_qcom_get_connected_tx_lanes(hba, &tx_lanes);  in ufs_qcom_link_startup_post_change()
198 		dev_err(hba->dev, "%s: ufs_qcom_phy_set_tx_lane_enable failed\n",  in ufs_qcom_link_startup_post_change()
205 static int ufs_qcom_check_hibern8(struct ufs_hba *hba)  in ufs_qcom_check_hibern8()  argument
212 		err = ufshcd_dme_get(hba,  in ufs_qcom_check_hibern8()
228 		err = ufshcd_dme_get(hba,  in ufs_qcom_check_hibern8()
234 		dev_err(hba->dev, "%s: unable to get TX_FSM_STATE, err %d\n",  in ufs_qcom_check_hibern8()
238 		dev_err(hba->dev, "%s: invalid TX_FSM_STATE = %d\n",  in ufs_qcom_check_hibern8()
247 	ufshcd_rmwl(host->hba, QUNIPRO_SEL,  in ufs_qcom_select_unipro_mode()
254 static int ufs_qcom_power_up_sequence(struct ufs_hba *hba)  in ufs_qcom_power_up_sequence()  argument
256 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_power_up_sequence()
263 	ufs_qcom_assert_reset(hba);  in ufs_qcom_power_up_sequence()
270 		dev_err(hba->dev,  in ufs_qcom_power_up_sequence()
277 	ufs_qcom_deassert_reset(hba);  in ufs_qcom_power_up_sequence()
286 		dev_err(hba->dev, "%s: ufs_qcom_phy_start_serdes() failed, ret = %d\n",  in ufs_qcom_power_up_sequence()
293 		dev_err(hba->dev,  in ufs_qcom_power_up_sequence()
311 static void ufs_qcom_enable_hw_clk_gating(struct ufs_hba *hba)  in ufs_qcom_enable_hw_clk_gating()  argument
313 	ufshcd_writel(hba,  in ufs_qcom_enable_hw_clk_gating()
314 		ufshcd_readl(hba, REG_UFS_CFG2) | REG_UFS_CFG2_CGC_EN_ALL,  in ufs_qcom_enable_hw_clk_gating()
321 static int ufs_qcom_hce_enable_notify(struct ufs_hba *hba,  in ufs_qcom_hce_enable_notify()  argument
324 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_hce_enable_notify()
329 		ufs_qcom_power_up_sequence(hba);  in ufs_qcom_hce_enable_notify()
339 		err = ufs_qcom_check_hibern8(hba);  in ufs_qcom_hce_enable_notify()
340 		ufs_qcom_enable_hw_clk_gating(hba);  in ufs_qcom_hce_enable_notify()
344 		dev_err(hba->dev, "%s: invalid status %d\n", __func__, status);  in ufs_qcom_hce_enable_notify()
354 static int ufs_qcom_cfg_timers(struct ufs_hba *hba, u32 gear,  in ufs_qcom_cfg_timers()  argument
358 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_cfg_timers()
391 	if (ufs_qcom_cap_qunipro(host) && !ufshcd_is_intr_aggr_allowed(hba))  in ufs_qcom_cfg_timers()
395 		dev_err(hba->dev, "%s: invalid gear = %d\n", __func__, gear);  in ufs_qcom_cfg_timers()
399 	list_for_each_entry(clki, &hba->clk_list_head, list) {  in ufs_qcom_cfg_timers()
409 	if (ufshcd_readl(hba, REG_UFS_SYS1CLK_1US) != core_clk_cycles_per_us) {  in ufs_qcom_cfg_timers()
410 		ufshcd_writel(hba, core_clk_cycles_per_us, REG_UFS_SYS1CLK_1US);  in ufs_qcom_cfg_timers()
430 				dev_err(hba->dev,  in ufs_qcom_cfg_timers()
439 				dev_err(hba->dev,  in ufs_qcom_cfg_timers()
447 			dev_err(hba->dev, "%s: invalid rate = %d\n",  in ufs_qcom_cfg_timers()
455 			dev_err(hba->dev,  in ufs_qcom_cfg_timers()
465 		dev_err(hba->dev, "%s: invalid mode = %d\n", __func__, hs);  in ufs_qcom_cfg_timers()
469 	if (ufshcd_readl(hba, REG_UFS_TX_SYMBOL_CLK_NS_US) !=  in ufs_qcom_cfg_timers()
472 		ufshcd_writel(hba, core_clk_period_in_ns | tx_clk_cycles_per_us,  in ufs_qcom_cfg_timers()
482 		ufshcd_writel(hba, ((core_clk_rate / MSEC_PER_SEC) * 100),  in ufs_qcom_cfg_timers()
498 static int ufs_qcom_link_startup_notify(struct ufs_hba *hba,  in ufs_qcom_link_startup_notify()  argument
502 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_link_startup_notify()
506 		if (ufs_qcom_cfg_timers(hba, UFS_PWM_G1, SLOWAUTO_MODE,  in ufs_qcom_link_startup_notify()
508 			dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n",  in ufs_qcom_link_startup_notify()
519 			err = ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba,  in ufs_qcom_link_startup_notify()
524 		ufs_qcom_link_startup_post_change(hba);  in ufs_qcom_link_startup_notify()
534 static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)  in ufs_qcom_suspend()  argument
536 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_suspend()
540 	if (ufs_qcom_is_link_off(hba)) {  in ufs_qcom_suspend()
550 		ufs_qcom_assert_reset(hba);  in ufs_qcom_suspend()
558 	if (!ufs_qcom_is_link_active(hba)) {  in ufs_qcom_suspend()
567 static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)  in ufs_qcom_resume()  argument
569 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_resume()
575 		dev_err(hba->dev, "%s: failed enabling regs, err = %d\n",  in ufs_qcom_resume()
584 	hba->is_sys_suspended = false;  in ufs_qcom_resume()
692 	struct device *dev = host->hba->dev;  in ufs_qcom_get_bus_vote()
750 			dev_err(host->hba->dev,  in ufs_qcom_set_bus_vote()
778 		dev_err(host->hba->dev, "%s: failed %d\n", __func__, err);  in ufs_qcom_update_bus_bw_vote()
788 	struct ufs_hba *hba = dev_get_drvdata(dev);  in show_ufs_to_mem_max_bus_bw()  local
789 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in show_ufs_to_mem_max_bus_bw()
799 	struct ufs_hba *hba = dev_get_drvdata(dev);  in store_ufs_to_mem_max_bus_bw()  local
800 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in store_ufs_to_mem_max_bus_bw()
815 	struct device *dev = host->hba->dev;  in ufs_qcom_bus_register()
908 static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba,  in ufs_qcom_pwr_change_notify()  argument
914 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_pwr_change_notify()
967 		if (ufs_qcom_cfg_timers(hba, dev_req_params->gear_rx,  in ufs_qcom_pwr_change_notify()
970 			dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n",  in ufs_qcom_pwr_change_notify()
983 			dev_err(hba->dev, "%s: ufs_qcom_phy_set_tx_lane_enable() failed res = %d\n",  in ufs_qcom_pwr_change_notify()
1001 static u32 ufs_qcom_get_ufs_hci_version(struct ufs_hba *hba)  in ufs_qcom_get_ufs_hci_version()  argument
1003 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_get_ufs_hci_version()
1020 static void ufs_qcom_advertise_quirks(struct ufs_hba *hba)  in ufs_qcom_advertise_quirks()  argument
1022 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_advertise_quirks()
1025 		hba->quirks |= UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS  in ufs_qcom_advertise_quirks()
1030 			hba->quirks |= UFSHCD_QUIRK_BROKEN_INTR_AGGR;  in ufs_qcom_advertise_quirks()
1032 		hba->quirks |= UFSHCD_QUIRK_BROKEN_LCC;  in ufs_qcom_advertise_quirks()
1036 		hba->quirks |= UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION;  in ufs_qcom_advertise_quirks()
1040 			hba->quirks |= (UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS  in ufs_qcom_advertise_quirks()
1046 static void ufs_qcom_set_caps(struct ufs_hba *hba)  in ufs_qcom_set_caps()  argument
1048 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_set_caps()
1050 	hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;  in ufs_qcom_set_caps()
1051 	hba->caps |= UFSHCD_CAP_CLK_SCALING;  in ufs_qcom_set_caps()
1052 	hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND;  in ufs_qcom_set_caps()
1067 static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on)  in ufs_qcom_setup_clocks()  argument
1069 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_setup_clocks()
1088 			dev_err(hba->dev, "%s enable phy ref clock failed, err=%d\n",  in ufs_qcom_setup_clocks()
1101 		if (!ufs_qcom_is_link_active(hba))  in ufs_qcom_setup_clocks()
1110 		dev_err(hba->dev, "%s: set bus vote failed %d\n",  in ufs_qcom_setup_clocks()
1139 static int ufs_qcom_init(struct ufs_hba *hba)  in ufs_qcom_init()  argument
1142 	struct device *dev = hba->dev;  in ufs_qcom_init()
1158 	host->hba = hba;  in ufs_qcom_init()
1159 	ufshcd_set_variant(hba, host);  in ufs_qcom_init()
1178 	ufs_qcom_get_controller_revision(hba, &host->hw_ver.major,  in ufs_qcom_init()
1186 		host->dev_ref_clk_ctrl_mmio = hba->mmio_base + REG_UFS_CFG1;  in ufs_qcom_init()
1218 	ufs_qcom_set_caps(hba);  in ufs_qcom_init()
1219 	ufs_qcom_advertise_quirks(hba);  in ufs_qcom_init()
1221 	ufs_qcom_setup_clocks(hba, true);  in ufs_qcom_init()
1223 	if (hba->dev->id < MAX_UFS_QCOM_HOSTS)  in ufs_qcom_init()
1224 		ufs_qcom_hosts[hba->dev->id] = host;  in ufs_qcom_init()
1243 	ufshcd_set_variant(hba, NULL);  in ufs_qcom_init()
1248 static void ufs_qcom_exit(struct ufs_hba *hba)  in ufs_qcom_exit()  argument
1250 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_exit()
1256 static int ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba *hba,  in ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div()  argument
1265 	err = ufshcd_dme_get(hba,  in ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div()
1277 	err = ufshcd_dme_set(hba,  in ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div()
1284 static int ufs_qcom_clk_scale_up_pre_change(struct ufs_hba *hba)  in ufs_qcom_clk_scale_up_pre_change()  argument
1290 static int ufs_qcom_clk_scale_up_post_change(struct ufs_hba *hba)  in ufs_qcom_clk_scale_up_post_change()  argument
1292 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_clk_scale_up_post_change()
1298 	return ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba, 150);  in ufs_qcom_clk_scale_up_post_change()
1301 static int ufs_qcom_clk_scale_down_pre_change(struct ufs_hba *hba)  in ufs_qcom_clk_scale_down_pre_change()  argument
1303 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_clk_scale_down_pre_change()
1310 	err = ufshcd_dme_get(hba,  in ufs_qcom_clk_scale_down_pre_change()
1318 		err = ufshcd_dme_set(hba,  in ufs_qcom_clk_scale_down_pre_change()
1326 static int ufs_qcom_clk_scale_down_post_change(struct ufs_hba *hba)  in ufs_qcom_clk_scale_down_post_change()  argument
1328 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_clk_scale_down_post_change()
1334 	return ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(hba, 75);  in ufs_qcom_clk_scale_down_post_change()
1337 static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba,  in ufs_qcom_clk_scale_notify()  argument
1340 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);  in ufs_qcom_clk_scale_notify()
1346 			err = ufs_qcom_clk_scale_up_pre_change(hba);  in ufs_qcom_clk_scale_notify()
1348 			err = ufs_qcom_clk_scale_down_pre_change(hba);  in ufs_qcom_clk_scale_notify()
1351 			err = ufs_qcom_clk_scale_up_post_change(hba);  in ufs_qcom_clk_scale_notify()
1353 			err = ufs_qcom_clk_scale_down_post_change(hba);  in ufs_qcom_clk_scale_notify()
1358 		ufs_qcom_cfg_timers(hba,  in ufs_qcom_clk_scale_notify()
1380 		dev_err(host->hba->dev,  in ufs_qcom_testbus_cfg_is_ok()
1392 		dev_err(host->hba->dev,  in ufs_qcom_testbus_cfg_is_ok()
1470 	pm_runtime_get_sync(host->hba->dev);  in ufs_qcom_testbus_config()
1471 	ufshcd_hold(host->hba, false);  in ufs_qcom_testbus_config()
1472 	ufshcd_rmwl(host->hba, TEST_BUS_SEL,  in ufs_qcom_testbus_config()
1475 	ufshcd_rmwl(host->hba, mask,  in ufs_qcom_testbus_config()
1478 	ufshcd_release(host->hba);  in ufs_qcom_testbus_config()
1479 	pm_runtime_put_sync(host->hba->dev);  in ufs_qcom_testbus_config()
1484 static void ufs_qcom_testbus_read(struct ufs_hba *hba)  in ufs_qcom_testbus_read()  argument
1486 	ufs_qcom_dump_regs(hba, UFS_TEST_BUS, 1, "UFS_TEST_BUS ");  in ufs_qcom_testbus_read()
1489 static void ufs_qcom_dump_dbg_regs(struct ufs_hba *hba)  in ufs_qcom_dump_dbg_regs()  argument
1491 	ufs_qcom_dump_regs(hba, REG_UFS_SYS1CLK_1US, 16,  in ufs_qcom_dump_dbg_regs()
1494 	ufs_qcom_testbus_read(hba);  in ufs_qcom_dump_dbg_regs()
1544 	struct ufs_hba *hba =  platform_get_drvdata(pdev);  in ufs_qcom_remove()  local
1547 	ufshcd_remove(hba);  in ufs_qcom_remove()