119 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
120 extern void r600_ih_ring_fini(struct radeon_device *rdev);
121 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
123 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
124 extern void sumo_rlc_fini(struct radeon_device *rdev);
125 extern int sumo_rlc_init(struct radeon_device *rdev);
126 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
127 extern void si_rlc_reset(struct radeon_device *rdev);
128 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
129 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
130 extern int cik_sdma_resume(struct radeon_device *rdev);
131 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
132 extern void cik_sdma_fini(struct radeon_device *rdev);
133 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
134 static void cik_rlc_stop(struct radeon_device *rdev);
135 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
136 static void cik_program_aspm(struct radeon_device *rdev);
137 static void cik_init_pg(struct radeon_device *rdev);
138 static void cik_init_cg(struct radeon_device *rdev);
139 static void cik_fini_pg(struct radeon_device *rdev);
140 static void cik_fini_cg(struct radeon_device *rdev);
141 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
154 int cik_get_allowed_info_register(struct radeon_device *rdev,  in cik_get_allowed_info_register()  argument
180 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)  in cik_didt_rreg()  argument
185 	spin_lock_irqsave(&rdev->didt_idx_lock, flags);  in cik_didt_rreg()
188 	spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);  in cik_didt_rreg()
192 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in cik_didt_wreg()  argument
196 	spin_lock_irqsave(&rdev->didt_idx_lock, flags);  in cik_didt_wreg()
199 	spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);  in cik_didt_wreg()
203 int ci_get_temp(struct radeon_device *rdev)  in ci_get_temp()  argument
222 int kv_get_temp(struct radeon_device *rdev)  in kv_get_temp()  argument
242 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)  in cik_pciep_rreg()  argument
247 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);  in cik_pciep_rreg()
251 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);  in cik_pciep_rreg()
255 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in cik_pciep_wreg()  argument
259 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);  in cik_pciep_wreg()
264 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);  in cik_pciep_wreg()
1624 static void cik_init_golden_registers(struct radeon_device *rdev)  in cik_init_golden_registers()  argument
1627 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_init_golden_registers()
1628 	switch (rdev->family) {  in cik_init_golden_registers()
1630 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1633 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1636 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1639 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1644 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1647 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1650 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1653 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1658 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1661 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1664 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1667 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1672 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1675 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1678 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1681 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1686 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1689 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1692 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1695 		radeon_program_register_sequence(rdev,  in cik_init_golden_registers()
1702 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_init_golden_registers()
1713 u32 cik_get_xclk(struct radeon_device *rdev)  in cik_get_xclk()  argument
1715         u32 reference_clock = rdev->clock.spll.reference_freq;  in cik_get_xclk()
1717 	if (rdev->flags & RADEON_IS_IGP) {  in cik_get_xclk()
1736 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)  in cik_mm_rdoorbell()  argument
1738 	if (index < rdev->doorbell.num_doorbells) {  in cik_mm_rdoorbell()
1739 		return readl(rdev->doorbell.ptr + index);  in cik_mm_rdoorbell()
1756 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)  in cik_mm_wdoorbell()  argument
1758 	if (index < rdev->doorbell.num_doorbells) {  in cik_mm_wdoorbell()
1759 		writel(v, rdev->doorbell.ptr + index);  in cik_mm_wdoorbell()
1849 static void cik_srbm_select(struct radeon_device *rdev,  in cik_srbm_select()  argument
1868 int ci_mc_load_microcode(struct radeon_device *rdev)  in ci_mc_load_microcode()  argument
1877 	if (!rdev->mc_fw)  in ci_mc_load_microcode()
1880 	if (rdev->new_fw) {  in ci_mc_load_microcode()
1882 			(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;  in ci_mc_load_microcode()
1888 			(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));  in ci_mc_load_microcode()
1891 			(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in ci_mc_load_microcode()
1893 		ucode_size = rdev->mc_fw->size / 4;  in ci_mc_load_microcode()
1895 		switch (rdev->family) {  in ci_mc_load_microcode()
1907 		fw_data = (const __be32 *)rdev->mc_fw->data;  in ci_mc_load_microcode()
1924 			if (rdev->new_fw) {  in ci_mc_load_microcode()
1934 		if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {  in ci_mc_load_microcode()
1943 			if (rdev->new_fw)  in ci_mc_load_microcode()
1955 		for (i = 0; i < rdev->usec_timeout; i++) {  in ci_mc_load_microcode()
1960 		for (i = 0; i < rdev->usec_timeout; i++) {  in ci_mc_load_microcode()
1982 static int cik_init_microcode(struct radeon_device *rdev)  in cik_init_microcode()  argument
1996 	switch (rdev->family) {  in cik_init_microcode()
2064 	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);  in cik_init_microcode()
2067 		err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);  in cik_init_microcode()
2070 		if (rdev->pfp_fw->size != pfp_req_size) {  in cik_init_microcode()
2073 			       rdev->pfp_fw->size, fw_name);  in cik_init_microcode()
2078 		err = radeon_ucode_validate(rdev->pfp_fw);  in cik_init_microcode()
2090 	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);  in cik_init_microcode()
2093 		err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);  in cik_init_microcode()
2096 		if (rdev->me_fw->size != me_req_size) {  in cik_init_microcode()
2099 			       rdev->me_fw->size, fw_name);  in cik_init_microcode()
2103 		err = radeon_ucode_validate(rdev->me_fw);  in cik_init_microcode()
2115 	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);  in cik_init_microcode()
2118 		err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);  in cik_init_microcode()
2121 		if (rdev->ce_fw->size != ce_req_size) {  in cik_init_microcode()
2124 			       rdev->ce_fw->size, fw_name);  in cik_init_microcode()
2128 		err = radeon_ucode_validate(rdev->ce_fw);  in cik_init_microcode()
2140 	err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);  in cik_init_microcode()
2143 		err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);  in cik_init_microcode()
2146 		if (rdev->mec_fw->size != mec_req_size) {  in cik_init_microcode()
2149 			       rdev->mec_fw->size, fw_name);  in cik_init_microcode()
2153 		err = radeon_ucode_validate(rdev->mec_fw);  in cik_init_microcode()
2164 	if (rdev->family == CHIP_KAVERI) {  in cik_init_microcode()
2166 		err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);  in cik_init_microcode()
2170 			err = radeon_ucode_validate(rdev->mec2_fw);  in cik_init_microcode()
2180 	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2183 		err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2186 		if (rdev->rlc_fw->size != rlc_req_size) {  in cik_init_microcode()
2189 			       rdev->rlc_fw->size, fw_name);  in cik_init_microcode()
2193 		err = radeon_ucode_validate(rdev->rlc_fw);  in cik_init_microcode()
2205 	err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);  in cik_init_microcode()
2208 		err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);  in cik_init_microcode()
2211 		if (rdev->sdma_fw->size != sdma_req_size) {  in cik_init_microcode()
2214 			       rdev->sdma_fw->size, fw_name);  in cik_init_microcode()
2218 		err = radeon_ucode_validate(rdev->sdma_fw);  in cik_init_microcode()
2230 	if (!(rdev->flags & RADEON_IS_IGP)) {  in cik_init_microcode()
2232 		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2235 			err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2238 				err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2242 			if ((rdev->mc_fw->size != mc_req_size) &&  in cik_init_microcode()
2243 			    (rdev->mc_fw->size != mc2_req_size)){  in cik_init_microcode()
2246 				       rdev->mc_fw->size, fw_name);  in cik_init_microcode()
2249 			DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);  in cik_init_microcode()
2251 			err = radeon_ucode_validate(rdev->mc_fw);  in cik_init_microcode()
2263 		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2266 			err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2271 				release_firmware(rdev->smc_fw);  in cik_init_microcode()
2272 				rdev->smc_fw = NULL;  in cik_init_microcode()
2274 			} else if (rdev->smc_fw->size != smc_req_size) {  in cik_init_microcode()
2277 				       rdev->smc_fw->size, fw_name);  in cik_init_microcode()
2281 			err = radeon_ucode_validate(rdev->smc_fw);  in cik_init_microcode()
2294 		rdev->new_fw = false;  in cik_init_microcode()
2299 		rdev->new_fw = true;  in cik_init_microcode()
2308 		release_firmware(rdev->pfp_fw);  in cik_init_microcode()
2309 		rdev->pfp_fw = NULL;  in cik_init_microcode()
2310 		release_firmware(rdev->me_fw);  in cik_init_microcode()
2311 		rdev->me_fw = NULL;  in cik_init_microcode()
2312 		release_firmware(rdev->ce_fw);  in cik_init_microcode()
2313 		rdev->ce_fw = NULL;  in cik_init_microcode()
2314 		release_firmware(rdev->mec_fw);  in cik_init_microcode()
2315 		rdev->mec_fw = NULL;  in cik_init_microcode()
2316 		release_firmware(rdev->mec2_fw);  in cik_init_microcode()
2317 		rdev->mec2_fw = NULL;  in cik_init_microcode()
2318 		release_firmware(rdev->rlc_fw);  in cik_init_microcode()
2319 		rdev->rlc_fw = NULL;  in cik_init_microcode()
2320 		release_firmware(rdev->sdma_fw);  in cik_init_microcode()
2321 		rdev->sdma_fw = NULL;  in cik_init_microcode()
2322 		release_firmware(rdev->mc_fw);  in cik_init_microcode()
2323 		rdev->mc_fw = NULL;  in cik_init_microcode()
2324 		release_firmware(rdev->smc_fw);  in cik_init_microcode()
2325 		rdev->smc_fw = NULL;  in cik_init_microcode()
2344 static void cik_tiling_mode_table_init(struct radeon_device *rdev)  in cik_tiling_mode_table_init()  argument
2350 	u32 num_rbs = rdev->config.cik.max_backends_per_se *  in cik_tiling_mode_table_init()
2351 		rdev->config.cik.max_shader_engines;  in cik_tiling_mode_table_init()
2353 	switch (rdev->config.cik.mem_row_size_in_kb) {  in cik_tiling_mode_table_init()
2366 	num_pipe_configs = rdev->config.cik.max_tile_pipes;  in cik_tiling_mode_table_init()
2497 			rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;  in cik_tiling_mode_table_init()
2590 			rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;  in cik_tiling_mode_table_init()
2720 			rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;  in cik_tiling_mode_table_init()
2813 			rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;  in cik_tiling_mode_table_init()
2944 				rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;  in cik_tiling_mode_table_init()
3074 				rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;  in cik_tiling_mode_table_init()
3168 			rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;  in cik_tiling_mode_table_init()
3298 			rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;  in cik_tiling_mode_table_init()
3391 			rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;  in cik_tiling_mode_table_init()
3409 static void cik_select_se_sh(struct radeon_device *rdev,  in cik_select_se_sh()  argument
3455 static u32 cik_get_rb_disabled(struct radeon_device *rdev,  in cik_get_rb_disabled()  argument
3485 static void cik_setup_rb(struct radeon_device *rdev,  in cik_setup_rb()  argument
3494 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_setup_rb()
3497 			cik_select_se_sh(rdev, i, j);  in cik_setup_rb()
3498 			data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);  in cik_setup_rb()
3499 			if (rdev->family == CHIP_HAWAII)  in cik_setup_rb()
3505 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_setup_rb()
3506 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_setup_rb()
3515 	rdev->config.cik.backend_enable_mask = enabled_rbs;  in cik_setup_rb()
3517 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_setup_rb()
3519 		cik_select_se_sh(rdev, i, 0xffffffff);  in cik_setup_rb()
3544 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_setup_rb()
3545 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_setup_rb()
3556 static void cik_gpu_init(struct radeon_device *rdev)  in cik_gpu_init()  argument
3564 	switch (rdev->family) {  in cik_gpu_init()
3566 		rdev->config.cik.max_shader_engines = 2;  in cik_gpu_init()
3567 		rdev->config.cik.max_tile_pipes = 4;  in cik_gpu_init()
3568 		rdev->config.cik.max_cu_per_sh = 7;  in cik_gpu_init()
3569 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3570 		rdev->config.cik.max_backends_per_se = 2;  in cik_gpu_init()
3571 		rdev->config.cik.max_texture_channel_caches = 4;  in cik_gpu_init()
3572 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3573 		rdev->config.cik.max_gs_threads = 32;  in cik_gpu_init()
3574 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3576 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3577 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3578 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3579 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3583 		rdev->config.cik.max_shader_engines = 4;  in cik_gpu_init()
3584 		rdev->config.cik.max_tile_pipes = 16;  in cik_gpu_init()
3585 		rdev->config.cik.max_cu_per_sh = 11;  in cik_gpu_init()
3586 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3587 		rdev->config.cik.max_backends_per_se = 4;  in cik_gpu_init()
3588 		rdev->config.cik.max_texture_channel_caches = 16;  in cik_gpu_init()
3589 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3590 		rdev->config.cik.max_gs_threads = 32;  in cik_gpu_init()
3591 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3593 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3594 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3595 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3596 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3600 		rdev->config.cik.max_shader_engines = 1;  in cik_gpu_init()
3601 		rdev->config.cik.max_tile_pipes = 4;  in cik_gpu_init()
3602 		if ((rdev->pdev->device == 0x1304) ||  in cik_gpu_init()
3603 		    (rdev->pdev->device == 0x1305) ||  in cik_gpu_init()
3604 		    (rdev->pdev->device == 0x130C) ||  in cik_gpu_init()
3605 		    (rdev->pdev->device == 0x130F) ||  in cik_gpu_init()
3606 		    (rdev->pdev->device == 0x1310) ||  in cik_gpu_init()
3607 		    (rdev->pdev->device == 0x1311) ||  in cik_gpu_init()
3608 		    (rdev->pdev->device == 0x131C)) {  in cik_gpu_init()
3609 			rdev->config.cik.max_cu_per_sh = 8;  in cik_gpu_init()
3610 			rdev->config.cik.max_backends_per_se = 2;  in cik_gpu_init()
3611 		} else if ((rdev->pdev->device == 0x1309) ||  in cik_gpu_init()
3612 			   (rdev->pdev->device == 0x130A) ||  in cik_gpu_init()
3613 			   (rdev->pdev->device == 0x130D) ||  in cik_gpu_init()
3614 			   (rdev->pdev->device == 0x1313) ||  in cik_gpu_init()
3615 			   (rdev->pdev->device == 0x131D)) {  in cik_gpu_init()
3616 			rdev->config.cik.max_cu_per_sh = 6;  in cik_gpu_init()
3617 			rdev->config.cik.max_backends_per_se = 2;  in cik_gpu_init()
3618 		} else if ((rdev->pdev->device == 0x1306) ||  in cik_gpu_init()
3619 			   (rdev->pdev->device == 0x1307) ||  in cik_gpu_init()
3620 			   (rdev->pdev->device == 0x130B) ||  in cik_gpu_init()
3621 			   (rdev->pdev->device == 0x130E) ||  in cik_gpu_init()
3622 			   (rdev->pdev->device == 0x1315) ||  in cik_gpu_init()
3623 			   (rdev->pdev->device == 0x1318) ||  in cik_gpu_init()
3624 			   (rdev->pdev->device == 0x131B)) {  in cik_gpu_init()
3625 			rdev->config.cik.max_cu_per_sh = 4;  in cik_gpu_init()
3626 			rdev->config.cik.max_backends_per_se = 1;  in cik_gpu_init()
3628 			rdev->config.cik.max_cu_per_sh = 3;  in cik_gpu_init()
3629 			rdev->config.cik.max_backends_per_se = 1;  in cik_gpu_init()
3631 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3632 		rdev->config.cik.max_texture_channel_caches = 4;  in cik_gpu_init()
3633 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3634 		rdev->config.cik.max_gs_threads = 16;  in cik_gpu_init()
3635 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3637 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3638 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3639 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3640 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3646 		rdev->config.cik.max_shader_engines = 1;  in cik_gpu_init()
3647 		rdev->config.cik.max_tile_pipes = 2;  in cik_gpu_init()
3648 		rdev->config.cik.max_cu_per_sh = 2;  in cik_gpu_init()
3649 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3650 		rdev->config.cik.max_backends_per_se = 1;  in cik_gpu_init()
3651 		rdev->config.cik.max_texture_channel_caches = 2;  in cik_gpu_init()
3652 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3653 		rdev->config.cik.max_gs_threads = 16;  in cik_gpu_init()
3654 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3656 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3657 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3658 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3659 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3682 	rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;  in cik_gpu_init()
3683 	rdev->config.cik.mem_max_burst_length_bytes = 256;  in cik_gpu_init()
3685 	rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;  in cik_gpu_init()
3686 	if (rdev->config.cik.mem_row_size_in_kb > 4)  in cik_gpu_init()
3687 		rdev->config.cik.mem_row_size_in_kb = 4;  in cik_gpu_init()
3689 	rdev->config.cik.shader_engine_tile_size = 32;  in cik_gpu_init()
3690 	rdev->config.cik.num_gpus = 1;  in cik_gpu_init()
3691 	rdev->config.cik.multi_gpu_tile_size = 64;  in cik_gpu_init()
3695 	switch (rdev->config.cik.mem_row_size_in_kb) {  in cik_gpu_init()
3715 	rdev->config.cik.tile_config = 0;  in cik_gpu_init()
3716 	switch (rdev->config.cik.num_tile_pipes) {  in cik_gpu_init()
3718 		rdev->config.cik.tile_config |= (0 << 0);  in cik_gpu_init()
3721 		rdev->config.cik.tile_config |= (1 << 0);  in cik_gpu_init()
3724 		rdev->config.cik.tile_config |= (2 << 0);  in cik_gpu_init()
3729 		rdev->config.cik.tile_config |= (3 << 0);  in cik_gpu_init()
3732 	rdev->config.cik.tile_config |=  in cik_gpu_init()
3734 	rdev->config.cik.tile_config |=  in cik_gpu_init()
3736 	rdev->config.cik.tile_config |=  in cik_gpu_init()
3748 	cik_tiling_mode_table_init(rdev);  in cik_gpu_init()
3750 	cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,  in cik_gpu_init()
3751 		     rdev->config.cik.max_sh_per_se,  in cik_gpu_init()
3752 		     rdev->config.cik.max_backends_per_se);  in cik_gpu_init()
3754 	rdev->config.cik.active_cus = 0;  in cik_gpu_init()
3755 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {  in cik_gpu_init()
3756 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {  in cik_gpu_init()
3757 			rdev->config.cik.active_cus +=  in cik_gpu_init()
3758 				hweight32(cik_get_cu_active_bitmap(rdev, i, j));  in cik_gpu_init()
3765 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_gpu_init()
3770 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_gpu_init()
3797 	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |  in cik_gpu_init()
3798 				 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |  in cik_gpu_init()
3799 				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |  in cik_gpu_init()
3800 				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));  in cik_gpu_init()
3826 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_gpu_init()
3844 static void cik_scratch_init(struct radeon_device *rdev)  in cik_scratch_init()  argument
3848 	rdev->scratch.num_reg = 7;  in cik_scratch_init()
3849 	rdev->scratch.reg_base = SCRATCH_REG0;  in cik_scratch_init()
3850 	for (i = 0; i < rdev->scratch.num_reg; i++) {  in cik_scratch_init()
3851 		rdev->scratch.free[i] = true;  in cik_scratch_init()
3852 		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);  in cik_scratch_init()
3867 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_ring_test()  argument
3874 	r = radeon_scratch_get(rdev, &scratch);  in cik_ring_test()
3880 	r = radeon_ring_lock(rdev, ring, 3);  in cik_ring_test()
3883 		radeon_scratch_free(rdev, scratch);  in cik_ring_test()
3889 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_ring_test()
3891 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_ring_test()
3897 	if (i < rdev->usec_timeout) {  in cik_ring_test()
3904 	radeon_scratch_free(rdev, scratch);  in cik_ring_test()
3916 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,  in cik_hdp_flush_cp_ring_emit()  argument
3919 	struct radeon_ring *ring = &rdev->ring[ridx];  in cik_hdp_flush_cp_ring_emit()
3962 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,  in cik_fence_gfx_ring_emit()  argument
3965 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cik_fence_gfx_ring_emit()
3966 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cik_fence_gfx_ring_emit()
4003 void cik_fence_compute_ring_emit(struct radeon_device *rdev,  in cik_fence_compute_ring_emit()  argument
4006 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cik_fence_compute_ring_emit()
4007 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cik_fence_compute_ring_emit()
4033 bool cik_semaphore_ring_emit(struct radeon_device *rdev,  in cik_semaphore_ring_emit()  argument
4067 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,  in cik_copy_cpdma()  argument
4074 	int ring_index = rdev->asic->copy.blit_ring_index;  in cik_copy_cpdma()
4075 	struct radeon_ring *ring = &rdev->ring[ring_index];  in cik_copy_cpdma()
4084 	r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);  in cik_copy_cpdma()
4087 		radeon_sync_free(rdev, &sync, NULL);  in cik_copy_cpdma()
4091 	radeon_sync_resv(rdev, &sync, resv, false);  in cik_copy_cpdma()
4092 	radeon_sync_rings(rdev, &sync, ring->idx);  in cik_copy_cpdma()
4113 	r = radeon_fence_emit(rdev, &fence, ring->idx);  in cik_copy_cpdma()
4115 		radeon_ring_unlock_undo(rdev, ring);  in cik_copy_cpdma()
4116 		radeon_sync_free(rdev, &sync, NULL);  in cik_copy_cpdma()
4120 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_copy_cpdma()
4121 	radeon_sync_free(rdev, &sync, fence);  in cik_copy_cpdma()
4141 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)  in cik_ring_ib_execute()  argument
4143 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in cik_ring_ib_execute()
4161 		} else if (rdev->wb.enabled) {  in cik_ring_ib_execute()
4191 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_ib_test()  argument
4199 	r = radeon_scratch_get(rdev, &scratch);  in cik_ib_test()
4205 	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);  in cik_ib_test()
4208 		radeon_scratch_free(rdev, scratch);  in cik_ib_test()
4215 	r = radeon_ib_schedule(rdev, &ib, NULL, false);  in cik_ib_test()
4217 		radeon_scratch_free(rdev, scratch);  in cik_ib_test()
4218 		radeon_ib_free(rdev, &ib);  in cik_ib_test()
4225 		radeon_scratch_free(rdev, scratch);  in cik_ib_test()
4226 		radeon_ib_free(rdev, &ib);  in cik_ib_test()
4229 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_ib_test()
4235 	if (i < rdev->usec_timeout) {  in cik_ib_test()
4242 	radeon_scratch_free(rdev, scratch);  in cik_ib_test()
4243 	radeon_ib_free(rdev, &ib);  in cik_ib_test()
4278 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)  in cik_cp_gfx_enable()  argument
4283 		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in cik_cp_gfx_enable()
4284 			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);  in cik_cp_gfx_enable()
4286 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cik_cp_gfx_enable()
4299 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)  in cik_cp_gfx_load_microcode()  argument
4303 	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)  in cik_cp_gfx_load_microcode()
4306 	cik_cp_gfx_enable(rdev, false);  in cik_cp_gfx_load_microcode()
4308 	if (rdev->new_fw) {  in cik_cp_gfx_load_microcode()
4310 			(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;  in cik_cp_gfx_load_microcode()
4312 			(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;  in cik_cp_gfx_load_microcode()
4314 			(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;  in cik_cp_gfx_load_microcode()
4324 			(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));  in cik_cp_gfx_load_microcode()
4333 			(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));  in cik_cp_gfx_load_microcode()
4342 			(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));  in cik_cp_gfx_load_microcode()
4353 		fw_data = (const __be32 *)rdev->pfp_fw->data;  in cik_cp_gfx_load_microcode()
4360 		fw_data = (const __be32 *)rdev->ce_fw->data;  in cik_cp_gfx_load_microcode()
4367 		fw_data = (const __be32 *)rdev->me_fw->data;  in cik_cp_gfx_load_microcode()
4386 static int cik_cp_gfx_start(struct radeon_device *rdev)  in cik_cp_gfx_start()  argument
4388 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_cp_gfx_start()
4392 	WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);  in cik_cp_gfx_start()
4396 	cik_cp_gfx_enable(rdev, true);  in cik_cp_gfx_start()
4398 	r = radeon_ring_lock(rdev, ring, cik_default_size + 17);  in cik_cp_gfx_start()
4433 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_cp_gfx_start()
4446 static void cik_cp_gfx_fini(struct radeon_device *rdev)  in cik_cp_gfx_fini()  argument
4448 	cik_cp_gfx_enable(rdev, false);  in cik_cp_gfx_fini()
4449 	radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);  in cik_cp_gfx_fini()
4461 static int cik_cp_gfx_resume(struct radeon_device *rdev)  in cik_cp_gfx_resume()  argument
4470 	if (rdev->family != CHIP_HAWAII)  in cik_cp_gfx_resume()
4479 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);  in cik_cp_gfx_resume()
4483 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_cp_gfx_resume()
4497 	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);  in cik_cp_gfx_resume()
4498 	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);  in cik_cp_gfx_resume()
4503 	if (!rdev->wb.enabled)  in cik_cp_gfx_resume()
4514 	cik_cp_gfx_start(rdev);  in cik_cp_gfx_resume()
4515 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;  in cik_cp_gfx_resume()
4516 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);  in cik_cp_gfx_resume()
4518 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cik_cp_gfx_resume()
4522 	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in cik_cp_gfx_resume()
4523 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);  in cik_cp_gfx_resume()
4528 u32 cik_gfx_get_rptr(struct radeon_device *rdev,  in cik_gfx_get_rptr()  argument
4533 	if (rdev->wb.enabled)  in cik_gfx_get_rptr()
4534 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in cik_gfx_get_rptr()
4541 u32 cik_gfx_get_wptr(struct radeon_device *rdev,  in cik_gfx_get_wptr()  argument
4551 void cik_gfx_set_wptr(struct radeon_device *rdev,  in cik_gfx_set_wptr()  argument
4558 u32 cik_compute_get_rptr(struct radeon_device *rdev,  in cik_compute_get_rptr()  argument
4563 	if (rdev->wb.enabled) {  in cik_compute_get_rptr()
4564 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in cik_compute_get_rptr()
4566 		mutex_lock(&rdev->srbm_mutex);  in cik_compute_get_rptr()
4567 		cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);  in cik_compute_get_rptr()
4569 		cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_compute_get_rptr()
4570 		mutex_unlock(&rdev->srbm_mutex);  in cik_compute_get_rptr()
4576 u32 cik_compute_get_wptr(struct radeon_device *rdev,  in cik_compute_get_wptr()  argument
4581 	if (rdev->wb.enabled) {  in cik_compute_get_wptr()
4583 		wptr = rdev->wb.wb[ring->wptr_offs/4];  in cik_compute_get_wptr()
4585 		mutex_lock(&rdev->srbm_mutex);  in cik_compute_get_wptr()
4586 		cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);  in cik_compute_get_wptr()
4588 		cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_compute_get_wptr()
4589 		mutex_unlock(&rdev->srbm_mutex);  in cik_compute_get_wptr()
4595 void cik_compute_set_wptr(struct radeon_device *rdev,  in cik_compute_set_wptr()  argument
4599 	rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;  in cik_compute_set_wptr()
4603 static void cik_compute_stop(struct radeon_device *rdev,  in cik_compute_stop()  argument
4608 	cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);  in cik_compute_stop()
4616 		for (j = 0; j < rdev->usec_timeout; j++) {  in cik_compute_stop()
4625 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_compute_stop()
4636 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)  in cik_cp_compute_enable()  argument
4645 		mutex_lock(&rdev->srbm_mutex);  in cik_cp_compute_enable()
4646 		cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);  in cik_cp_compute_enable()
4647 		cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);  in cik_cp_compute_enable()
4648 		mutex_unlock(&rdev->srbm_mutex);  in cik_cp_compute_enable()
4651 		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;  in cik_cp_compute_enable()
4652 		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;  in cik_cp_compute_enable()
4665 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)  in cik_cp_compute_load_microcode()  argument
4669 	if (!rdev->mec_fw)  in cik_cp_compute_load_microcode()
4672 	cik_cp_compute_enable(rdev, false);  in cik_cp_compute_load_microcode()
4674 	if (rdev->new_fw) {  in cik_cp_compute_load_microcode()
4676 			(const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;  in cik_cp_compute_load_microcode()
4684 			(rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));  in cik_cp_compute_load_microcode()
4692 		if (rdev->family == CHIP_KAVERI) {  in cik_cp_compute_load_microcode()
4694 				(const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;  in cik_cp_compute_load_microcode()
4697 				(rdev->mec2_fw->data +  in cik_cp_compute_load_microcode()
4709 		fw_data = (const __be32 *)rdev->mec_fw->data;  in cik_cp_compute_load_microcode()
4715 		if (rdev->family == CHIP_KAVERI) {  in cik_cp_compute_load_microcode()
4717 			fw_data = (const __be32 *)rdev->mec_fw->data;  in cik_cp_compute_load_microcode()
4736 static int cik_cp_compute_start(struct radeon_device *rdev)  in cik_cp_compute_start()  argument
4738 	cik_cp_compute_enable(rdev, true);  in cik_cp_compute_start()
4751 static void cik_cp_compute_fini(struct radeon_device *rdev)  in cik_cp_compute_fini()  argument
4755 	cik_cp_compute_enable(rdev, false);  in cik_cp_compute_fini()
4763 		if (rdev->ring[idx].mqd_obj) {  in cik_cp_compute_fini()
4764 			r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);  in cik_cp_compute_fini()
4766 				dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);  in cik_cp_compute_fini()
4768 			radeon_bo_unpin(rdev->ring[idx].mqd_obj);  in cik_cp_compute_fini()
4769 			radeon_bo_unreserve(rdev->ring[idx].mqd_obj);  in cik_cp_compute_fini()
4771 			radeon_bo_unref(&rdev->ring[idx].mqd_obj);  in cik_cp_compute_fini()
4772 			rdev->ring[idx].mqd_obj = NULL;  in cik_cp_compute_fini()
4777 static void cik_mec_fini(struct radeon_device *rdev)  in cik_mec_fini()  argument
4781 	if (rdev->mec.hpd_eop_obj) {  in cik_mec_fini()
4782 		r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);  in cik_mec_fini()
4784 			dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);  in cik_mec_fini()
4785 		radeon_bo_unpin(rdev->mec.hpd_eop_obj);  in cik_mec_fini()
4786 		radeon_bo_unreserve(rdev->mec.hpd_eop_obj);  in cik_mec_fini()
4788 		radeon_bo_unref(&rdev->mec.hpd_eop_obj);  in cik_mec_fini()
4789 		rdev->mec.hpd_eop_obj = NULL;  in cik_mec_fini()
4795 static int cik_mec_init(struct radeon_device *rdev)  in cik_mec_init()  argument
4806 	rdev->mec.num_mec = 1;  in cik_mec_init()
4807 	rdev->mec.num_pipe = 1;  in cik_mec_init()
4808 	rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;  in cik_mec_init()
4810 	if (rdev->mec.hpd_eop_obj == NULL) {  in cik_mec_init()
4811 		r = radeon_bo_create(rdev,  in cik_mec_init()
4812 				     rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,  in cik_mec_init()
4815 				     &rdev->mec.hpd_eop_obj);  in cik_mec_init()
4817 			dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);  in cik_mec_init()
4822 	r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);  in cik_mec_init()
4824 		cik_mec_fini(rdev);  in cik_mec_init()
4827 	r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,  in cik_mec_init()
4828 			  &rdev->mec.hpd_eop_gpu_addr);  in cik_mec_init()
4830 		dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);  in cik_mec_init()
4831 		cik_mec_fini(rdev);  in cik_mec_init()
4834 	r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);  in cik_mec_init()
4836 		dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);  in cik_mec_init()
4837 		cik_mec_fini(rdev);  in cik_mec_init()
4842 	memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);  in cik_mec_init()
4844 	radeon_bo_kunmap(rdev->mec.hpd_eop_obj);  in cik_mec_init()
4845 	radeon_bo_unreserve(rdev->mec.hpd_eop_obj);  in cik_mec_init()
4926 static int cik_cp_compute_resume(struct radeon_device *rdev)  in cik_cp_compute_resume()  argument
4938 	r = cik_cp_compute_start(rdev);  in cik_cp_compute_resume()
4948 	mutex_lock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4950 	eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;  in cik_cp_compute_resume()
4952 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_cp_compute_resume()
4967 	mutex_unlock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4976 		if (rdev->ring[idx].mqd_obj == NULL) {  in cik_cp_compute_resume()
4977 			r = radeon_bo_create(rdev,  in cik_cp_compute_resume()
4981 					     NULL, &rdev->ring[idx].mqd_obj);  in cik_cp_compute_resume()
4983 				dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);  in cik_cp_compute_resume()
4988 		r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);  in cik_cp_compute_resume()
4990 			cik_cp_compute_fini(rdev);  in cik_cp_compute_resume()
4993 		r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,  in cik_cp_compute_resume()
4996 			dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);  in cik_cp_compute_resume()
4997 			cik_cp_compute_fini(rdev);  in cik_cp_compute_resume()
5000 		r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);  in cik_cp_compute_resume()
5002 			dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);  in cik_cp_compute_resume()
5003 			cik_cp_compute_fini(rdev);  in cik_cp_compute_resume()
5017 		mutex_lock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
5018 		cik_srbm_select(rdev, rdev->ring[idx].me,  in cik_cp_compute_resume()
5019 				rdev->ring[idx].pipe,  in cik_cp_compute_resume()
5020 				rdev->ring[idx].queue, 0);  in cik_cp_compute_resume()
5043 			for (j = 0; j < rdev->usec_timeout; j++) {  in cik_cp_compute_resume()
5064 		hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;  in cik_cp_compute_resume()
5076 			order_base_2(rdev->ring[idx].ring_size / 8);  in cik_cp_compute_resume()
5090 			wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;  in cik_cp_compute_resume()
5092 			wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;  in cik_cp_compute_resume()
5101 			wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;  in cik_cp_compute_resume()
5103 			wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;  in cik_cp_compute_resume()
5118 				DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);  in cik_cp_compute_resume()
5130 		rdev->ring[idx].wptr = 0;  in cik_cp_compute_resume()
5131 		mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;  in cik_cp_compute_resume()
5143 		cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_cp_compute_resume()
5144 		mutex_unlock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
5146 		radeon_bo_kunmap(rdev->ring[idx].mqd_obj);  in cik_cp_compute_resume()
5147 		radeon_bo_unreserve(rdev->ring[idx].mqd_obj);  in cik_cp_compute_resume()
5149 		rdev->ring[idx].ready = true;  in cik_cp_compute_resume()
5150 		r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);  in cik_cp_compute_resume()
5152 			rdev->ring[idx].ready = false;  in cik_cp_compute_resume()
5158 static void cik_cp_enable(struct radeon_device *rdev, bool enable)  in cik_cp_enable()  argument
5160 	cik_cp_gfx_enable(rdev, enable);  in cik_cp_enable()
5161 	cik_cp_compute_enable(rdev, enable);  in cik_cp_enable()
5164 static int cik_cp_load_microcode(struct radeon_device *rdev)  in cik_cp_load_microcode()  argument
5168 	r = cik_cp_gfx_load_microcode(rdev);  in cik_cp_load_microcode()
5171 	r = cik_cp_compute_load_microcode(rdev);  in cik_cp_load_microcode()
5178 static void cik_cp_fini(struct radeon_device *rdev)  in cik_cp_fini()  argument
5180 	cik_cp_gfx_fini(rdev);  in cik_cp_fini()
5181 	cik_cp_compute_fini(rdev);  in cik_cp_fini()
5184 static int cik_cp_resume(struct radeon_device *rdev)  in cik_cp_resume()  argument
5188 	cik_enable_gui_idle_interrupt(rdev, false);  in cik_cp_resume()
5190 	r = cik_cp_load_microcode(rdev);  in cik_cp_resume()
5194 	r = cik_cp_gfx_resume(rdev);  in cik_cp_resume()
5197 	r = cik_cp_compute_resume(rdev);  in cik_cp_resume()
5201 	cik_enable_gui_idle_interrupt(rdev, true);  in cik_cp_resume()
5206 static void cik_print_gpu_status_regs(struct radeon_device *rdev)  in cik_print_gpu_status_regs()  argument
5208 	dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",  in cik_print_gpu_status_regs()
5210 	dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",  in cik_print_gpu_status_regs()
5212 	dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",  in cik_print_gpu_status_regs()
5214 	dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",  in cik_print_gpu_status_regs()
5216 	dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",  in cik_print_gpu_status_regs()
5218 	dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",  in cik_print_gpu_status_regs()
5220 	dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",  in cik_print_gpu_status_regs()
5222 	dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",  in cik_print_gpu_status_regs()
5224 	dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",  in cik_print_gpu_status_regs()
5226 	dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",  in cik_print_gpu_status_regs()
5228 	dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));  in cik_print_gpu_status_regs()
5229 	dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",  in cik_print_gpu_status_regs()
5231 	dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",  in cik_print_gpu_status_regs()
5233 	dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",  in cik_print_gpu_status_regs()
5235 	dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",  in cik_print_gpu_status_regs()
5237 	dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",  in cik_print_gpu_status_regs()
5239 	dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));  in cik_print_gpu_status_regs()
5240 	dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));  in cik_print_gpu_status_regs()
5241 	dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",  in cik_print_gpu_status_regs()
5243 	dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));  in cik_print_gpu_status_regs()
5255 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)  in cik_gpu_check_soft_reset()  argument
5315 	if (evergreen_is_display_hung(rdev))  in cik_gpu_check_soft_reset()
5335 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)  in cik_gpu_soft_reset()  argument
5344 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);  in cik_gpu_soft_reset()
5346 	cik_print_gpu_status_regs(rdev);  in cik_gpu_soft_reset()
5347 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in cik_gpu_soft_reset()
5349 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in cik_gpu_soft_reset()
5353 	cik_fini_pg(rdev);  in cik_gpu_soft_reset()
5354 	cik_fini_cg(rdev);  in cik_gpu_soft_reset()
5357 	cik_rlc_stop(rdev);  in cik_gpu_soft_reset()
5378 	evergreen_mc_stop(rdev, &save);  in cik_gpu_soft_reset()
5379 	if (evergreen_mc_wait_for_idle(rdev)) {  in cik_gpu_soft_reset()
5380 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cik_gpu_soft_reset()
5416 	if (!(rdev->flags & RADEON_IS_IGP)) {  in cik_gpu_soft_reset()
5424 		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);  in cik_gpu_soft_reset()
5438 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in cik_gpu_soft_reset()
5452 	evergreen_mc_resume(rdev, &save);  in cik_gpu_soft_reset()
5455 	cik_print_gpu_status_regs(rdev);  in cik_gpu_soft_reset()
5464 static void kv_save_regs_for_reset(struct radeon_device *rdev,  in kv_save_regs_for_reset()  argument
5476 static void kv_restore_regs_for_reset(struct radeon_device *rdev,  in kv_restore_regs_for_reset()  argument
5549 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)  in cik_gpu_pci_config_reset()  argument
5555 	dev_info(rdev->dev, "GPU pci config reset\n");  in cik_gpu_pci_config_reset()
5560 	cik_fini_pg(rdev);  in cik_gpu_pci_config_reset()
5561 	cik_fini_cg(rdev);  in cik_gpu_pci_config_reset()
5580 	cik_rlc_stop(rdev);  in cik_gpu_pci_config_reset()
5585 	evergreen_mc_stop(rdev, &save);  in cik_gpu_pci_config_reset()
5586 	if (evergreen_mc_wait_for_idle(rdev)) {  in cik_gpu_pci_config_reset()
5587 		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");  in cik_gpu_pci_config_reset()
5590 	if (rdev->flags & RADEON_IS_IGP)  in cik_gpu_pci_config_reset()
5591 		kv_save_regs_for_reset(rdev, &kv_save);  in cik_gpu_pci_config_reset()
5594 	pci_clear_master(rdev->pdev);  in cik_gpu_pci_config_reset()
5596 	radeon_pci_config_reset(rdev);  in cik_gpu_pci_config_reset()
5601 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_gpu_pci_config_reset()
5608 	if (rdev->flags & RADEON_IS_IGP)  in cik_gpu_pci_config_reset()
5609 		kv_restore_regs_for_reset(rdev, &kv_save);  in cik_gpu_pci_config_reset()
5621 int cik_asic_reset(struct radeon_device *rdev)  in cik_asic_reset()  argument
5625 	reset_mask = cik_gpu_check_soft_reset(rdev);  in cik_asic_reset()
5628 		r600_set_bios_scratch_engine_hung(rdev, true);  in cik_asic_reset()
5631 	cik_gpu_soft_reset(rdev, reset_mask);  in cik_asic_reset()
5633 	reset_mask = cik_gpu_check_soft_reset(rdev);  in cik_asic_reset()
5637 		cik_gpu_pci_config_reset(rdev);  in cik_asic_reset()
5639 	reset_mask = cik_gpu_check_soft_reset(rdev);  in cik_asic_reset()
5642 		r600_set_bios_scratch_engine_hung(rdev, false);  in cik_asic_reset()
5656 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_gfx_is_lockup()  argument
5658 	u32 reset_mask = cik_gpu_check_soft_reset(rdev);  in cik_gfx_is_lockup()
5663 		radeon_ring_lockup_update(rdev, ring);  in cik_gfx_is_lockup()
5666 	return radeon_ring_test_lockup(rdev, ring);  in cik_gfx_is_lockup()
5678 static void cik_mc_program(struct radeon_device *rdev)  in cik_mc_program()  argument
5694 	evergreen_mc_stop(rdev, &save);  in cik_mc_program()
5695 	if (radeon_mc_wait_for_idle(rdev)) {  in cik_mc_program()
5696 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cik_mc_program()
5702 	       rdev->mc.vram_start >> 12);  in cik_mc_program()
5704 	       rdev->mc.vram_end >> 12);  in cik_mc_program()
5706 	       rdev->vram_scratch.gpu_addr >> 12);  in cik_mc_program()
5707 	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;  in cik_mc_program()
5708 	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);  in cik_mc_program()
5711 	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));  in cik_mc_program()
5717 	if (radeon_mc_wait_for_idle(rdev)) {  in cik_mc_program()
5718 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cik_mc_program()
5720 	evergreen_mc_resume(rdev, &save);  in cik_mc_program()
5723 	rv515_vga_render_disable(rdev);  in cik_mc_program()
5735 static int cik_mc_init(struct radeon_device *rdev)  in cik_mc_init()  argument
5741 	rdev->mc.vram_is_ddr = true;  in cik_mc_init()
5779 	rdev->mc.vram_width = numchan * chansize;  in cik_mc_init()
5781 	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);  in cik_mc_init()
5782 	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);  in cik_mc_init()
5784 	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in cik_mc_init()
5785 	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in cik_mc_init()
5786 	rdev->mc.visible_vram_size = rdev->mc.aper_size;  in cik_mc_init()
5787 	si_vram_gtt_location(rdev, &rdev->mc);  in cik_mc_init()
5788 	radeon_update_bandwidth_info(rdev);  in cik_mc_init()
5806 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)  in cik_pcie_gart_tlb_flush()  argument
5815 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)  in cik_pcie_init_compute_vmid()  argument
5824 	mutex_lock(&rdev->srbm_mutex);  in cik_pcie_init_compute_vmid()
5826 		cik_srbm_select(rdev, 0, 0, 0, i);  in cik_pcie_init_compute_vmid()
5833 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_pcie_init_compute_vmid()
5834 	mutex_unlock(&rdev->srbm_mutex);  in cik_pcie_init_compute_vmid()
5848 static int cik_pcie_gart_enable(struct radeon_device *rdev)  in cik_pcie_gart_enable()  argument
5852 	if (rdev->gart.robj == NULL) {  in cik_pcie_gart_enable()
5853 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");  in cik_pcie_gart_enable()
5856 	r = radeon_gart_table_vram_pin(rdev);  in cik_pcie_gart_enable()
5879 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);  in cik_pcie_gart_enable()
5880 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);  in cik_pcie_gart_enable()
5881 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);  in cik_pcie_gart_enable()
5883 			(u32)(rdev->dummy_page.addr >> 12));  in cik_pcie_gart_enable()
5895 	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);  in cik_pcie_gart_enable()
5899 			       rdev->vm_manager.saved_table_addr[i]);  in cik_pcie_gart_enable()
5902 			       rdev->vm_manager.saved_table_addr[i]);  in cik_pcie_gart_enable()
5907 	       (u32)(rdev->dummy_page.addr >> 12));  in cik_pcie_gart_enable()
5924 	if (rdev->family == CHIP_KAVERI) {  in cik_pcie_gart_enable()
5932 	mutex_lock(&rdev->srbm_mutex);  in cik_pcie_gart_enable()
5934 		cik_srbm_select(rdev, 0, 0, 0, i);  in cik_pcie_gart_enable()
5947 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_pcie_gart_enable()
5948 	mutex_unlock(&rdev->srbm_mutex);  in cik_pcie_gart_enable()
5950 	cik_pcie_init_compute_vmid(rdev);  in cik_pcie_gart_enable()
5952 	cik_pcie_gart_tlb_flush(rdev);  in cik_pcie_gart_enable()
5954 		 (unsigned)(rdev->mc.gtt_size >> 20),  in cik_pcie_gart_enable()
5955 		 (unsigned long long)rdev->gart.table_addr);  in cik_pcie_gart_enable()
5956 	rdev->gart.ready = true;  in cik_pcie_gart_enable()
5967 static void cik_pcie_gart_disable(struct radeon_device *rdev)  in cik_pcie_gart_disable()  argument
5977 		rdev->vm_manager.saved_table_addr[i] = RREG32(reg);  in cik_pcie_gart_disable()
5996 	radeon_gart_table_vram_unpin(rdev);  in cik_pcie_gart_disable()
6006 static void cik_pcie_gart_fini(struct radeon_device *rdev)  in cik_pcie_gart_fini()  argument
6008 	cik_pcie_gart_disable(rdev);  in cik_pcie_gart_fini()
6009 	radeon_gart_table_vram_free(rdev);  in cik_pcie_gart_fini()
6010 	radeon_gart_fini(rdev);  in cik_pcie_gart_fini()
6022 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)  in cik_ib_parse()  argument
6042 int cik_vm_init(struct radeon_device *rdev)  in cik_vm_init()  argument
6050 	rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;  in cik_vm_init()
6052 	if (rdev->flags & RADEON_IS_IGP) {  in cik_vm_init()
6055 		rdev->vm_manager.vram_base_offset = tmp;  in cik_vm_init()
6057 		rdev->vm_manager.vram_base_offset = 0;  in cik_vm_init()
6069 void cik_vm_fini(struct radeon_device *rdev)  in cik_vm_fini()  argument
6082 static void cik_vm_decode_fault(struct radeon_device *rdev,  in cik_vm_decode_fault()  argument
6091 	if (rdev->family == CHIP_HAWAII)  in cik_vm_decode_fault()
6110 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,  in cik_vm_flush()  argument
6155 	cik_hdp_flush_cp_ring_emit(rdev, ring->idx);  in cik_vm_flush()
6190 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,  in cik_enable_gui_idle_interrupt()  argument
6202 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)  in cik_enable_lbpw()  argument
6214 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)  in cik_wait_for_rlc_serdes()  argument
6219 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_wait_for_rlc_serdes()
6220 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {  in cik_wait_for_rlc_serdes()
6221 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {  in cik_wait_for_rlc_serdes()
6222 			cik_select_se_sh(rdev, i, j);  in cik_wait_for_rlc_serdes()
6223 			for (k = 0; k < rdev->usec_timeout; k++) {  in cik_wait_for_rlc_serdes()
6230 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_wait_for_rlc_serdes()
6231 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_wait_for_rlc_serdes()
6234 	for (k = 0; k < rdev->usec_timeout; k++) {  in cik_wait_for_rlc_serdes()
6241 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)  in cik_update_rlc()  argument
6250 static u32 cik_halt_rlc(struct radeon_device *rdev)  in cik_halt_rlc()  argument
6262 		for (i = 0; i < rdev->usec_timeout; i++) {  in cik_halt_rlc()
6268 		cik_wait_for_rlc_serdes(rdev);  in cik_halt_rlc()
6274 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)  in cik_enter_rlc_safe_mode()  argument
6282 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_enter_rlc_safe_mode()
6288 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_enter_rlc_safe_mode()
6295 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)  in cik_exit_rlc_safe_mode()  argument
6310 static void cik_rlc_stop(struct radeon_device *rdev)  in cik_rlc_stop()  argument
6314 	cik_enable_gui_idle_interrupt(rdev, false);  in cik_rlc_stop()
6316 	cik_wait_for_rlc_serdes(rdev);  in cik_rlc_stop()
6326 static void cik_rlc_start(struct radeon_device *rdev)  in cik_rlc_start()  argument
6330 	cik_enable_gui_idle_interrupt(rdev, true);  in cik_rlc_start()
6344 static int cik_rlc_resume(struct radeon_device *rdev)  in cik_rlc_resume()  argument
6348 	if (!rdev->rlc_fw)  in cik_rlc_resume()
6351 	cik_rlc_stop(rdev);  in cik_rlc_resume()
6357 	si_rlc_reset(rdev);  in cik_rlc_resume()
6359 	cik_init_pg(rdev);  in cik_rlc_resume()
6361 	cik_init_cg(rdev);  in cik_rlc_resume()
6366 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_rlc_resume()
6367 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_rlc_resume()
6371 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_rlc_resume()
6376 	if (rdev->new_fw) {  in cik_rlc_resume()
6378 			(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;  in cik_rlc_resume()
6380 			(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_rlc_resume()
6392 		switch (rdev->family) {  in cik_rlc_resume()
6409 		fw_data = (const __be32 *)rdev->rlc_fw->data;  in cik_rlc_resume()
6417 	cik_enable_lbpw(rdev, false);  in cik_rlc_resume()
6419 	if (rdev->family == CHIP_BONAIRE)  in cik_rlc_resume()
6422 	cik_rlc_start(rdev);  in cik_rlc_resume()
6427 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)  in cik_enable_cgcg()  argument
6433 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {  in cik_enable_cgcg()
6434 		cik_enable_gui_idle_interrupt(rdev, true);  in cik_enable_cgcg()
6436 		tmp = cik_halt_rlc(rdev);  in cik_enable_cgcg()
6438 		mutex_lock(&rdev->grbm_idx_mutex);  in cik_enable_cgcg()
6439 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_enable_cgcg()
6444 		mutex_unlock(&rdev->grbm_idx_mutex);  in cik_enable_cgcg()
6446 		cik_update_rlc(rdev, tmp);  in cik_enable_cgcg()
6450 		cik_enable_gui_idle_interrupt(rdev, false);  in cik_enable_cgcg()
6465 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)  in cik_enable_mgcg()  argument
6469 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {  in cik_enable_mgcg()
6470 		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {  in cik_enable_mgcg()
6471 			if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {  in cik_enable_mgcg()
6485 		tmp = cik_halt_rlc(rdev);  in cik_enable_mgcg()
6487 		mutex_lock(&rdev->grbm_idx_mutex);  in cik_enable_mgcg()
6488 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_enable_mgcg()
6493 		mutex_unlock(&rdev->grbm_idx_mutex);  in cik_enable_mgcg()
6495 		cik_update_rlc(rdev, tmp);  in cik_enable_mgcg()
6497 		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {  in cik_enable_mgcg()
6503 			if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&  in cik_enable_mgcg()
6504 			    (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))  in cik_enable_mgcg()
6535 		tmp = cik_halt_rlc(rdev);  in cik_enable_mgcg()
6537 		mutex_lock(&rdev->grbm_idx_mutex);  in cik_enable_mgcg()
6538 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_enable_mgcg()
6543 		mutex_unlock(&rdev->grbm_idx_mutex);  in cik_enable_mgcg()
6545 		cik_update_rlc(rdev, tmp);  in cik_enable_mgcg()
6562 static void cik_enable_mc_ls(struct radeon_device *rdev,  in cik_enable_mc_ls()  argument
6570 		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))  in cik_enable_mc_ls()
6579 static void cik_enable_mc_mgcg(struct radeon_device *rdev,  in cik_enable_mc_mgcg()  argument
6587 		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))  in cik_enable_mc_mgcg()
6596 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,  in cik_enable_sdma_mgcg()  argument
6601 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {  in cik_enable_sdma_mgcg()
6617 static void cik_enable_sdma_mgls(struct radeon_device *rdev,  in cik_enable_sdma_mgls()  argument
6622 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {  in cik_enable_sdma_mgls()
6645 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,  in cik_enable_uvd_mgcg()  argument
6650 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {  in cik_enable_uvd_mgcg()
6671 static void cik_enable_bif_mgls(struct radeon_device *rdev,  in cik_enable_bif_mgls()  argument
6678 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))  in cik_enable_bif_mgls()
6689 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,  in cik_enable_hdp_mgcg()  argument
6696 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))  in cik_enable_hdp_mgcg()
6705 static void cik_enable_hdp_ls(struct radeon_device *rdev,  in cik_enable_hdp_ls()  argument
6712 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))  in cik_enable_hdp_ls()
6721 void cik_update_cg(struct radeon_device *rdev,  in cik_update_cg()  argument
6726 		cik_enable_gui_idle_interrupt(rdev, false);  in cik_update_cg()
6729 			cik_enable_mgcg(rdev, true);  in cik_update_cg()
6730 			cik_enable_cgcg(rdev, true);  in cik_update_cg()
6732 			cik_enable_cgcg(rdev, false);  in cik_update_cg()
6733 			cik_enable_mgcg(rdev, false);  in cik_update_cg()
6735 		cik_enable_gui_idle_interrupt(rdev, true);  in cik_update_cg()
6739 		if (!(rdev->flags & RADEON_IS_IGP)) {  in cik_update_cg()
6740 			cik_enable_mc_mgcg(rdev, enable);  in cik_update_cg()
6741 			cik_enable_mc_ls(rdev, enable);  in cik_update_cg()
6746 		cik_enable_sdma_mgcg(rdev, enable);  in cik_update_cg()
6747 		cik_enable_sdma_mgls(rdev, enable);  in cik_update_cg()
6751 		cik_enable_bif_mgls(rdev, enable);  in cik_update_cg()
6755 		if (rdev->has_uvd)  in cik_update_cg()
6756 			cik_enable_uvd_mgcg(rdev, enable);  in cik_update_cg()
6760 		cik_enable_hdp_mgcg(rdev, enable);  in cik_update_cg()
6761 		cik_enable_hdp_ls(rdev, enable);  in cik_update_cg()
6765 		vce_v2_0_enable_mgcg(rdev, enable);  in cik_update_cg()
6769 static void cik_init_cg(struct radeon_device *rdev)  in cik_init_cg()  argument
6772 	cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);  in cik_init_cg()
6774 	if (rdev->has_uvd)  in cik_init_cg()
6775 		si_init_uvd_internal_cg(rdev);  in cik_init_cg()
6777 	cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |  in cik_init_cg()
6784 static void cik_fini_cg(struct radeon_device *rdev)  in cik_fini_cg()  argument
6786 	cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |  in cik_fini_cg()
6792 	cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);  in cik_fini_cg()
6795 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,  in cik_enable_sck_slowdown_on_pu()  argument
6801 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))  in cik_enable_sck_slowdown_on_pu()
6809 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,  in cik_enable_sck_slowdown_on_pd()  argument
6815 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))  in cik_enable_sck_slowdown_on_pd()
6823 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)  in cik_enable_cp_pg()  argument
6828 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))  in cik_enable_cp_pg()
6836 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)  in cik_enable_gds_pg()  argument
6841 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))  in cik_enable_gds_pg()
6853 void cik_init_cp_pg_table(struct radeon_device *rdev)  in cik_init_cp_pg_table()  argument
6860 	if (rdev->family == CHIP_KAVERI)  in cik_init_cp_pg_table()
6863 	if (rdev->rlc.cp_table_ptr == NULL)  in cik_init_cp_pg_table()
6867 	dst_ptr = rdev->rlc.cp_table_ptr;  in cik_init_cp_pg_table()
6869 		if (rdev->new_fw) {  in cik_init_cp_pg_table()
6874 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;  in cik_init_cp_pg_table()
6876 					(rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6880 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;  in cik_init_cp_pg_table()
6882 					(rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6886 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;  in cik_init_cp_pg_table()
6888 					(rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6892 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;  in cik_init_cp_pg_table()
6894 					(rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6898 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;  in cik_init_cp_pg_table()
6900 					(rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6915 				fw_data = (const __be32 *)rdev->ce_fw->data;  in cik_init_cp_pg_table()
6918 				fw_data = (const __be32 *)rdev->pfp_fw->data;  in cik_init_cp_pg_table()
6921 				fw_data = (const __be32 *)rdev->me_fw->data;  in cik_init_cp_pg_table()
6924 				fw_data = (const __be32 *)rdev->mec_fw->data;  in cik_init_cp_pg_table()
6937 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,  in cik_enable_gfx_cgpg()  argument
6942 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {  in cik_enable_gfx_cgpg()
6967 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)  in cik_get_cu_active_bitmap()  argument
6972 	mutex_lock(&rdev->grbm_idx_mutex);  in cik_get_cu_active_bitmap()
6973 	cik_select_se_sh(rdev, se, sh);  in cik_get_cu_active_bitmap()
6976 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_get_cu_active_bitmap()
6977 	mutex_unlock(&rdev->grbm_idx_mutex);  in cik_get_cu_active_bitmap()
6984 	for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {  in cik_get_cu_active_bitmap()
6992 static void cik_init_ao_cu_mask(struct radeon_device *rdev)  in cik_init_ao_cu_mask()  argument
6998 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {  in cik_init_ao_cu_mask()
6999 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {  in cik_init_ao_cu_mask()
7003 			for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {  in cik_init_ao_cu_mask()
7004 				if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {  in cik_init_ao_cu_mask()
7025 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,  in cik_enable_gfx_static_mgpg()  argument
7031 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))  in cik_enable_gfx_static_mgpg()
7039 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,  in cik_enable_gfx_dynamic_mgpg()  argument
7045 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))  in cik_enable_gfx_dynamic_mgpg()
7056 static void cik_init_gfx_cgpg(struct radeon_device *rdev)  in cik_init_gfx_cgpg()  argument
7061 	if (rdev->rlc.cs_data) {  in cik_init_gfx_cgpg()
7063 		WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));  in cik_init_gfx_cgpg()
7064 		WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));  in cik_init_gfx_cgpg()
7065 		WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);  in cik_init_gfx_cgpg()
7071 	if (rdev->rlc.reg_list) {  in cik_init_gfx_cgpg()
7073 		for (i = 0; i < rdev->rlc.reg_list_size; i++)  in cik_init_gfx_cgpg()
7074 			WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);  in cik_init_gfx_cgpg()
7082 	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);  in cik_init_gfx_cgpg()
7083 	WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);  in cik_init_gfx_cgpg()
7105 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)  in cik_update_gfx_pg()  argument
7107 	cik_enable_gfx_cgpg(rdev, enable);  in cik_update_gfx_pg()
7108 	cik_enable_gfx_static_mgpg(rdev, enable);  in cik_update_gfx_pg()
7109 	cik_enable_gfx_dynamic_mgpg(rdev, enable);  in cik_update_gfx_pg()
7112 u32 cik_get_csb_size(struct radeon_device *rdev)  in cik_get_csb_size()  argument
7118 	if (rdev->rlc.cs_data == NULL)  in cik_get_csb_size()
7126 	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {  in cik_get_csb_size()
7144 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)  in cik_get_csb_buffer()  argument
7150 	if (rdev->rlc.cs_data == NULL)  in cik_get_csb_buffer()
7162 	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {  in cik_get_csb_buffer()
7178 	switch (rdev->family) {  in cik_get_csb_buffer()
7209 static void cik_init_pg(struct radeon_device *rdev)  in cik_init_pg()  argument
7211 	if (rdev->pg_flags) {  in cik_init_pg()
7212 		cik_enable_sck_slowdown_on_pu(rdev, true);  in cik_init_pg()
7213 		cik_enable_sck_slowdown_on_pd(rdev, true);  in cik_init_pg()
7214 		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {  in cik_init_pg()
7215 			cik_init_gfx_cgpg(rdev);  in cik_init_pg()
7216 			cik_enable_cp_pg(rdev, true);  in cik_init_pg()
7217 			cik_enable_gds_pg(rdev, true);  in cik_init_pg()
7219 		cik_init_ao_cu_mask(rdev);  in cik_init_pg()
7220 		cik_update_gfx_pg(rdev, true);  in cik_init_pg()
7224 static void cik_fini_pg(struct radeon_device *rdev)  in cik_fini_pg()  argument
7226 	if (rdev->pg_flags) {  in cik_fini_pg()
7227 		cik_update_gfx_pg(rdev, false);  in cik_fini_pg()
7228 		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {  in cik_fini_pg()
7229 			cik_enable_cp_pg(rdev, false);  in cik_fini_pg()
7230 			cik_enable_gds_pg(rdev, false);  in cik_fini_pg()
7257 static void cik_enable_interrupts(struct radeon_device *rdev)  in cik_enable_interrupts()  argument
7266 	rdev->ih.enabled = true;  in cik_enable_interrupts()
7276 static void cik_disable_interrupts(struct radeon_device *rdev)  in cik_disable_interrupts()  argument
7288 	rdev->ih.enabled = false;  in cik_disable_interrupts()
7289 	rdev->ih.rptr = 0;  in cik_disable_interrupts()
7299 static void cik_disable_interrupt_state(struct radeon_device *rdev)  in cik_disable_interrupt_state()  argument
7328 	if (rdev->num_crtc >= 4) {  in cik_disable_interrupt_state()
7332 	if (rdev->num_crtc >= 6) {  in cik_disable_interrupt_state()
7337 	if (rdev->num_crtc >= 2) {  in cik_disable_interrupt_state()
7341 	if (rdev->num_crtc >= 4) {  in cik_disable_interrupt_state()
7345 	if (rdev->num_crtc >= 6) {  in cik_disable_interrupt_state()
7380 static int cik_irq_init(struct radeon_device *rdev)  in cik_irq_init()  argument
7387 	ret = r600_ih_ring_alloc(rdev);  in cik_irq_init()
7392 	cik_disable_interrupts(rdev);  in cik_irq_init()
7395 	ret = cik_rlc_resume(rdev);  in cik_irq_init()
7397 		r600_ih_ring_fini(rdev);  in cik_irq_init()
7403 	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);  in cik_irq_init()
7413 	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);  in cik_irq_init()
7414 	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);  in cik_irq_init()
7420 	if (rdev->wb.enabled)  in cik_irq_init()
7424 	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);  in cik_irq_init()
7425 	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);  in cik_irq_init()
7436 	if (rdev->msi_enabled)  in cik_irq_init()
7441 	cik_disable_interrupt_state(rdev);  in cik_irq_init()
7443 	pci_set_master(rdev->pdev);  in cik_irq_init()
7446 	cik_enable_interrupts(rdev);  in cik_irq_init()
7460 int cik_irq_set(struct radeon_device *rdev)  in cik_irq_set()  argument
7469 	if (!rdev->irq.installed) {  in cik_irq_set()
7474 	if (!rdev->ih.enabled) {  in cik_irq_set()
7475 		cik_disable_interrupts(rdev);  in cik_irq_set()
7477 		cik_disable_interrupt_state(rdev);  in cik_irq_set()
7498 	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {  in cik_irq_set()
7502 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {  in cik_irq_set()
7503 		struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_irq_set()
7518 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {  in cik_irq_set()
7519 		struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_irq_set()
7535 	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {  in cik_irq_set()
7540 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {  in cik_irq_set()
7545 	if (rdev->irq.crtc_vblank_int[0] ||  in cik_irq_set()
7546 	    atomic_read(&rdev->irq.pflip[0])) {  in cik_irq_set()
7550 	if (rdev->irq.crtc_vblank_int[1] ||  in cik_irq_set()
7551 	    atomic_read(&rdev->irq.pflip[1])) {  in cik_irq_set()
7555 	if (rdev->irq.crtc_vblank_int[2] ||  in cik_irq_set()
7556 	    atomic_read(&rdev->irq.pflip[2])) {  in cik_irq_set()
7560 	if (rdev->irq.crtc_vblank_int[3] ||  in cik_irq_set()
7561 	    atomic_read(&rdev->irq.pflip[3])) {  in cik_irq_set()
7565 	if (rdev->irq.crtc_vblank_int[4] ||  in cik_irq_set()
7566 	    atomic_read(&rdev->irq.pflip[4])) {  in cik_irq_set()
7570 	if (rdev->irq.crtc_vblank_int[5] ||  in cik_irq_set()
7571 	    atomic_read(&rdev->irq.pflip[5])) {  in cik_irq_set()
7575 	if (rdev->irq.hpd[0]) {  in cik_irq_set()
7579 	if (rdev->irq.hpd[1]) {  in cik_irq_set()
7583 	if (rdev->irq.hpd[2]) {  in cik_irq_set()
7587 	if (rdev->irq.hpd[3]) {  in cik_irq_set()
7591 	if (rdev->irq.hpd[4]) {  in cik_irq_set()
7595 	if (rdev->irq.hpd[5]) {  in cik_irq_set()
7611 	if (rdev->num_crtc >= 4) {  in cik_irq_set()
7615 	if (rdev->num_crtc >= 6) {  in cik_irq_set()
7620 	if (rdev->num_crtc >= 2) {  in cik_irq_set()
7626 	if (rdev->num_crtc >= 4) {  in cik_irq_set()
7632 	if (rdev->num_crtc >= 6) {  in cik_irq_set()
7661 static inline void cik_irq_ack(struct radeon_device *rdev)  in cik_irq_ack()  argument
7665 	rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);  in cik_irq_ack()
7666 	rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);  in cik_irq_ack()
7667 	rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);  in cik_irq_ack()
7668 	rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);  in cik_irq_ack()
7669 	rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);  in cik_irq_ack()
7670 	rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);  in cik_irq_ack()
7671 	rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);  in cik_irq_ack()
7673 	rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7675 	rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7677 	if (rdev->num_crtc >= 4) {  in cik_irq_ack()
7678 		rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7680 		rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7683 	if (rdev->num_crtc >= 6) {  in cik_irq_ack()
7684 		rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7686 		rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7690 	if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7693 	if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7696 	if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)  in cik_irq_ack()
7698 	if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)  in cik_irq_ack()
7700 	if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)  in cik_irq_ack()
7702 	if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)  in cik_irq_ack()
7705 	if (rdev->num_crtc >= 4) {  in cik_irq_ack()
7706 		if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7709 		if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7712 		if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)  in cik_irq_ack()
7714 		if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)  in cik_irq_ack()
7716 		if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)  in cik_irq_ack()
7718 		if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)  in cik_irq_ack()
7722 	if (rdev->num_crtc >= 6) {  in cik_irq_ack()
7723 		if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7726 		if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7729 		if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)  in cik_irq_ack()
7731 		if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)  in cik_irq_ack()
7733 		if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)  in cik_irq_ack()
7735 		if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)  in cik_irq_ack()
7739 	if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {  in cik_irq_ack()
7744 	if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {  in cik_irq_ack()
7749 	if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {  in cik_irq_ack()
7754 	if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {  in cik_irq_ack()
7759 	if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {  in cik_irq_ack()
7764 	if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {  in cik_irq_ack()
7769 	if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {  in cik_irq_ack()
7774 	if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {  in cik_irq_ack()
7779 	if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {  in cik_irq_ack()
7784 	if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {  in cik_irq_ack()
7789 	if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {  in cik_irq_ack()
7794 	if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {  in cik_irq_ack()
7808 static void cik_irq_disable(struct radeon_device *rdev)  in cik_irq_disable()  argument
7810 	cik_disable_interrupts(rdev);  in cik_irq_disable()
7813 	cik_irq_ack(rdev);  in cik_irq_disable()
7814 	cik_disable_interrupt_state(rdev);  in cik_irq_disable()
7825 static void cik_irq_suspend(struct radeon_device *rdev)  in cik_irq_suspend()  argument
7827 	cik_irq_disable(rdev);  in cik_irq_suspend()
7828 	cik_rlc_stop(rdev);  in cik_irq_suspend()
7840 static void cik_irq_fini(struct radeon_device *rdev)  in cik_irq_fini()  argument
7842 	cik_irq_suspend(rdev);  in cik_irq_fini()
7843 	r600_ih_ring_fini(rdev);  in cik_irq_fini()
7857 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)  in cik_get_ih_wptr()  argument
7861 	if (rdev->wb.enabled)  in cik_get_ih_wptr()
7862 		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);  in cik_get_ih_wptr()
7872 		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",  in cik_get_ih_wptr()
7873 			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);  in cik_get_ih_wptr()
7874 		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;  in cik_get_ih_wptr()
7879 	return (wptr & rdev->ih.ptr_mask);  in cik_get_ih_wptr()
7914 int cik_irq_process(struct radeon_device *rdev)  in cik_irq_process()  argument
7916 	struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_irq_process()
7917 	struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_irq_process()
7929 	if (!rdev->ih.enabled || rdev->shutdown)  in cik_irq_process()
7932 	wptr = cik_get_ih_wptr(rdev);  in cik_irq_process()
7936 	if (atomic_xchg(&rdev->ih.lock, 1))  in cik_irq_process()
7939 	rptr = rdev->ih.rptr;  in cik_irq_process()
7946 	cik_irq_ack(rdev);  in cik_irq_process()
7952 		radeon_kfd_interrupt(rdev,  in cik_irq_process()
7953 				(const void *) &rdev->ih.ring[ring_index]);  in cik_irq_process()
7955 		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;  in cik_irq_process()
7956 		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;  in cik_irq_process()
7957 		ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;  in cik_irq_process()
7963 				if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))  in cik_irq_process()
7966 				if (rdev->irq.crtc_vblank_int[0]) {  in cik_irq_process()
7967 					drm_handle_vblank(rdev->ddev, 0);  in cik_irq_process()
7968 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7969 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7971 				if (atomic_read(&rdev->irq.pflip[0]))  in cik_irq_process()
7972 					radeon_crtc_handle_vblank(rdev, 0);  in cik_irq_process()
7973 				rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;  in cik_irq_process()
7978 				if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))  in cik_irq_process()
7981 				rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;  in cik_irq_process()
7993 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))  in cik_irq_process()
7996 				if (rdev->irq.crtc_vblank_int[1]) {  in cik_irq_process()
7997 					drm_handle_vblank(rdev->ddev, 1);  in cik_irq_process()
7998 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7999 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
8001 				if (atomic_read(&rdev->irq.pflip[1]))  in cik_irq_process()
8002 					radeon_crtc_handle_vblank(rdev, 1);  in cik_irq_process()
8003 				rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;  in cik_irq_process()
8008 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))  in cik_irq_process()
8011 				rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;  in cik_irq_process()
8023 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))  in cik_irq_process()
8026 				if (rdev->irq.crtc_vblank_int[2]) {  in cik_irq_process()
8027 					drm_handle_vblank(rdev->ddev, 2);  in cik_irq_process()
8028 					rdev->pm.vblank_sync = true;  in cik_irq_process()
8029 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
8031 				if (atomic_read(&rdev->irq.pflip[2]))  in cik_irq_process()
8032 					radeon_crtc_handle_vblank(rdev, 2);  in cik_irq_process()
8033 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;  in cik_irq_process()
8038 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))  in cik_irq_process()
8041 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;  in cik_irq_process()
8053 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))  in cik_irq_process()
8056 				if (rdev->irq.crtc_vblank_int[3]) {  in cik_irq_process()
8057 					drm_handle_vblank(rdev->ddev, 3);  in cik_irq_process()
8058 					rdev->pm.vblank_sync = true;  in cik_irq_process()
8059 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
8061 				if (atomic_read(&rdev->irq.pflip[3]))  in cik_irq_process()
8062 					radeon_crtc_handle_vblank(rdev, 3);  in cik_irq_process()
8063 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;  in cik_irq_process()
8068 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))  in cik_irq_process()
8071 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;  in cik_irq_process()
8083 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))  in cik_irq_process()
8086 				if (rdev->irq.crtc_vblank_int[4]) {  in cik_irq_process()
8087 					drm_handle_vblank(rdev->ddev, 4);  in cik_irq_process()
8088 					rdev->pm.vblank_sync = true;  in cik_irq_process()
8089 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
8091 				if (atomic_read(&rdev->irq.pflip[4]))  in cik_irq_process()
8092 					radeon_crtc_handle_vblank(rdev, 4);  in cik_irq_process()
8093 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;  in cik_irq_process()
8098 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))  in cik_irq_process()
8101 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;  in cik_irq_process()
8113 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))  in cik_irq_process()
8116 				if (rdev->irq.crtc_vblank_int[5]) {  in cik_irq_process()
8117 					drm_handle_vblank(rdev->ddev, 5);  in cik_irq_process()
8118 					rdev->pm.vblank_sync = true;  in cik_irq_process()
8119 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
8121 				if (atomic_read(&rdev->irq.pflip[5]))  in cik_irq_process()
8122 					radeon_crtc_handle_vblank(rdev, 5);  in cik_irq_process()
8123 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;  in cik_irq_process()
8128 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))  in cik_irq_process()
8131 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;  in cik_irq_process()
8148 				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);  in cik_irq_process()
8153 				if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))  in cik_irq_process()
8156 				rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;  in cik_irq_process()
8162 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))  in cik_irq_process()
8165 				rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;  in cik_irq_process()
8171 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))  in cik_irq_process()
8174 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;  in cik_irq_process()
8180 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))  in cik_irq_process()
8183 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;  in cik_irq_process()
8189 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))  in cik_irq_process()
8192 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;  in cik_irq_process()
8198 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))  in cik_irq_process()
8201 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;  in cik_irq_process()
8207 				if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))  in cik_irq_process()
8210 				rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;  in cik_irq_process()
8216 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))  in cik_irq_process()
8219 				rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;  in cik_irq_process()
8225 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))  in cik_irq_process()
8228 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;  in cik_irq_process()
8234 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))  in cik_irq_process()
8237 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;  in cik_irq_process()
8243 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))  in cik_irq_process()
8246 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;  in cik_irq_process()
8252 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))  in cik_irq_process()
8255 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;  in cik_irq_process()
8271 			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);  in cik_irq_process()
8282 			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);  in cik_irq_process()
8283 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in cik_irq_process()
8285 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in cik_irq_process()
8287 			cik_vm_decode_fault(rdev, status, addr, mc_client);  in cik_irq_process()
8293 				radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);  in cik_irq_process()
8296 				radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);  in cik_irq_process()
8305 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in cik_irq_process()
8315 				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in cik_irq_process()
8320 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);  in cik_irq_process()
8322 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);  in cik_irq_process()
8381 					radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);  in cik_irq_process()
8394 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);  in cik_irq_process()
8408 			rdev->pm.dpm.thermal.high_to_low = false;  in cik_irq_process()
8413 			rdev->pm.dpm.thermal.high_to_low = true;  in cik_irq_process()
8465 		rptr &= rdev->ih.ptr_mask;  in cik_irq_process()
8469 		schedule_work(&rdev->dp_work);  in cik_irq_process()
8471 		schedule_delayed_work(&rdev->hotplug_work, 0);  in cik_irq_process()
8473 		rdev->needs_reset = true;  in cik_irq_process()
8474 		wake_up_all(&rdev->fence_queue);  in cik_irq_process()
8477 		schedule_work(&rdev->pm.dpm.thermal.work);  in cik_irq_process()
8478 	rdev->ih.rptr = rptr;  in cik_irq_process()
8479 	atomic_set(&rdev->ih.lock, 0);  in cik_irq_process()
8482 	wptr = cik_get_ih_wptr(rdev);  in cik_irq_process()
8501 static int cik_startup(struct radeon_device *rdev)  in cik_startup()  argument
8508 	cik_pcie_gen3_enable(rdev);  in cik_startup()
8510 	cik_program_aspm(rdev);  in cik_startup()
8513 	r = r600_vram_scratch_init(rdev);  in cik_startup()
8517 	cik_mc_program(rdev);  in cik_startup()
8519 	if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {  in cik_startup()
8520 		r = ci_mc_load_microcode(rdev);  in cik_startup()
8527 	r = cik_pcie_gart_enable(rdev);  in cik_startup()
8530 	cik_gpu_init(rdev);  in cik_startup()
8533 	if (rdev->flags & RADEON_IS_IGP) {  in cik_startup()
8534 		if (rdev->family == CHIP_KAVERI) {  in cik_startup()
8535 			rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;  in cik_startup()
8536 			rdev->rlc.reg_list_size =  in cik_startup()
8539 			rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;  in cik_startup()
8540 			rdev->rlc.reg_list_size =  in cik_startup()
8544 	rdev->rlc.cs_data = ci_cs_data;  in cik_startup()
8545 	rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;  in cik_startup()
8546 	r = sumo_rlc_init(rdev);  in cik_startup()
8553 	r = radeon_wb_init(rdev);  in cik_startup()
8558 	r = cik_mec_init(rdev);  in cik_startup()
8564 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);  in cik_startup()
8566 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cik_startup()
8570 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);  in cik_startup()
8572 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cik_startup()
8576 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);  in cik_startup()
8578 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cik_startup()
8582 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);  in cik_startup()
8584 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in cik_startup()
8588 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);  in cik_startup()
8590 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in cik_startup()
8594 	r = radeon_uvd_resume(rdev);  in cik_startup()
8596 		r = uvd_v4_2_resume(rdev);  in cik_startup()
8598 			r = radeon_fence_driver_start_ring(rdev,  in cik_startup()
8601 				dev_err(rdev->dev, "UVD fences init error (%d).\n", r);  in cik_startup()
8605 		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;  in cik_startup()
8607 	r = radeon_vce_resume(rdev);  in cik_startup()
8609 		r = vce_v2_0_resume(rdev);  in cik_startup()
8611 			r = radeon_fence_driver_start_ring(rdev,  in cik_startup()
8614 			r = radeon_fence_driver_start_ring(rdev,  in cik_startup()
8618 		dev_err(rdev->dev, "VCE init error (%d).\n", r);  in cik_startup()
8619 		rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;  in cik_startup()
8620 		rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;  in cik_startup()
8624 	if (!rdev->irq.installed) {  in cik_startup()
8625 		r = radeon_irq_kms_init(rdev);  in cik_startup()
8630 	r = cik_irq_init(rdev);  in cik_startup()
8633 		radeon_irq_kms_fini(rdev);  in cik_startup()
8636 	cik_irq_set(rdev);  in cik_startup()
8638 	if (rdev->family == CHIP_HAWAII) {  in cik_startup()
8639 		if (rdev->new_fw)  in cik_startup()
8647 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_startup()
8648 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,  in cik_startup()
8655 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_startup()
8656 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,  in cik_startup()
8666 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_startup()
8667 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,  in cik_startup()
8677 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cik_startup()
8678 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,  in cik_startup()
8683 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cik_startup()
8684 	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,  in cik_startup()
8689 	r = cik_cp_resume(rdev);  in cik_startup()
8693 	r = cik_sdma_resume(rdev);  in cik_startup()
8697 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];  in cik_startup()
8699 		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,  in cik_startup()
8702 			r = uvd_v1_0_init(rdev);  in cik_startup()
8709 	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];  in cik_startup()
8711 		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,  in cik_startup()
8714 	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];  in cik_startup()
8716 		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,  in cik_startup()
8720 		r = vce_v1_0_init(rdev);  in cik_startup()
8724 	r = radeon_ib_pool_init(rdev);  in cik_startup()
8726 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);  in cik_startup()
8730 	r = radeon_vm_manager_init(rdev);  in cik_startup()
8732 		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);  in cik_startup()
8736 	r = radeon_audio_init(rdev);  in cik_startup()
8740 	r = radeon_kfd_resume(rdev);  in cik_startup()
8756 int cik_resume(struct radeon_device *rdev)  in cik_resume()  argument
8761 	atom_asic_init(rdev->mode_info.atom_context);  in cik_resume()
8764 	cik_init_golden_registers(rdev);  in cik_resume()
8766 	if (rdev->pm.pm_method == PM_METHOD_DPM)  in cik_resume()
8767 		radeon_pm_resume(rdev);  in cik_resume()
8769 	rdev->accel_working = true;  in cik_resume()
8770 	r = cik_startup(rdev);  in cik_resume()
8773 		rdev->accel_working = false;  in cik_resume()
8790 int cik_suspend(struct radeon_device *rdev)  in cik_suspend()  argument
8792 	radeon_kfd_suspend(rdev);  in cik_suspend()
8793 	radeon_pm_suspend(rdev);  in cik_suspend()
8794 	radeon_audio_fini(rdev);  in cik_suspend()
8795 	radeon_vm_manager_fini(rdev);  in cik_suspend()
8796 	cik_cp_enable(rdev, false);  in cik_suspend()
8797 	cik_sdma_enable(rdev, false);  in cik_suspend()
8798 	uvd_v1_0_fini(rdev);  in cik_suspend()
8799 	radeon_uvd_suspend(rdev);  in cik_suspend()
8800 	radeon_vce_suspend(rdev);  in cik_suspend()
8801 	cik_fini_pg(rdev);  in cik_suspend()
8802 	cik_fini_cg(rdev);  in cik_suspend()
8803 	cik_irq_suspend(rdev);  in cik_suspend()
8804 	radeon_wb_disable(rdev);  in cik_suspend()
8805 	cik_pcie_gart_disable(rdev);  in cik_suspend()
8825 int cik_init(struct radeon_device *rdev)  in cik_init()  argument
8831 	if (!radeon_get_bios(rdev)) {  in cik_init()
8832 		if (ASIC_IS_AVIVO(rdev))  in cik_init()
8836 	if (!rdev->is_atom_bios) {  in cik_init()
8837 		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");  in cik_init()
8840 	r = radeon_atombios_init(rdev);  in cik_init()
8845 	if (!radeon_card_posted(rdev)) {  in cik_init()
8846 		if (!rdev->bios) {  in cik_init()
8847 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");  in cik_init()
8851 		atom_asic_init(rdev->mode_info.atom_context);  in cik_init()
8854 	cik_init_golden_registers(rdev);  in cik_init()
8856 	cik_scratch_init(rdev);  in cik_init()
8858 	radeon_surface_init(rdev);  in cik_init()
8860 	radeon_get_clock_info(rdev->ddev);  in cik_init()
8863 	r = radeon_fence_driver_init(rdev);  in cik_init()
8868 	r = cik_mc_init(rdev);  in cik_init()
8872 	r = radeon_bo_init(rdev);  in cik_init()
8876 	if (rdev->flags & RADEON_IS_IGP) {  in cik_init()
8877 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||  in cik_init()
8878 		    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {  in cik_init()
8879 			r = cik_init_microcode(rdev);  in cik_init()
8886 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||  in cik_init()
8887 		    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||  in cik_init()
8888 		    !rdev->mc_fw) {  in cik_init()
8889 			r = cik_init_microcode(rdev);  in cik_init()
8898 	radeon_pm_init(rdev);  in cik_init()
8900 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_init()
8902 	r600_ring_init(rdev, ring, 1024 * 1024);  in cik_init()
8904 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_init()
8906 	r600_ring_init(rdev, ring, 1024 * 1024);  in cik_init()
8907 	r = radeon_doorbell_get(rdev, &ring->doorbell_index);  in cik_init()
8911 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_init()
8913 	r600_ring_init(rdev, ring, 1024 * 1024);  in cik_init()
8914 	r = radeon_doorbell_get(rdev, &ring->doorbell_index);  in cik_init()
8918 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cik_init()
8920 	r600_ring_init(rdev, ring, 256 * 1024);  in cik_init()
8922 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cik_init()
8924 	r600_ring_init(rdev, ring, 256 * 1024);  in cik_init()
8926 	r = radeon_uvd_init(rdev);  in cik_init()
8928 		ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];  in cik_init()
8930 		r600_ring_init(rdev, ring, 4096);  in cik_init()
8933 	r = radeon_vce_init(rdev);  in cik_init()
8935 		ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];  in cik_init()
8937 		r600_ring_init(rdev, ring, 4096);  in cik_init()
8939 		ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];  in cik_init()
8941 		r600_ring_init(rdev, ring, 4096);  in cik_init()
8944 	rdev->ih.ring_obj = NULL;  in cik_init()
8945 	r600_ih_ring_init(rdev, 64 * 1024);  in cik_init()
8947 	r = r600_pcie_gart_init(rdev);  in cik_init()
8951 	rdev->accel_working = true;  in cik_init()
8952 	r = cik_startup(rdev);  in cik_init()
8954 		dev_err(rdev->dev, "disabling GPU acceleration\n");  in cik_init()
8955 		cik_cp_fini(rdev);  in cik_init()
8956 		cik_sdma_fini(rdev);  in cik_init()
8957 		cik_irq_fini(rdev);  in cik_init()
8958 		sumo_rlc_fini(rdev);  in cik_init()
8959 		cik_mec_fini(rdev);  in cik_init()
8960 		radeon_wb_fini(rdev);  in cik_init()
8961 		radeon_ib_pool_fini(rdev);  in cik_init()
8962 		radeon_vm_manager_fini(rdev);  in cik_init()
8963 		radeon_irq_kms_fini(rdev);  in cik_init()
8964 		cik_pcie_gart_fini(rdev);  in cik_init()
8965 		rdev->accel_working = false;  in cik_init()
8972 	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {  in cik_init()
8989 void cik_fini(struct radeon_device *rdev)  in cik_fini()  argument
8991 	radeon_pm_fini(rdev);  in cik_fini()
8992 	cik_cp_fini(rdev);  in cik_fini()
8993 	cik_sdma_fini(rdev);  in cik_fini()
8994 	cik_fini_pg(rdev);  in cik_fini()
8995 	cik_fini_cg(rdev);  in cik_fini()
8996 	cik_irq_fini(rdev);  in cik_fini()
8997 	sumo_rlc_fini(rdev);  in cik_fini()
8998 	cik_mec_fini(rdev);  in cik_fini()
8999 	radeon_wb_fini(rdev);  in cik_fini()
9000 	radeon_vm_manager_fini(rdev);  in cik_fini()
9001 	radeon_ib_pool_fini(rdev);  in cik_fini()
9002 	radeon_irq_kms_fini(rdev);  in cik_fini()
9003 	uvd_v1_0_fini(rdev);  in cik_fini()
9004 	radeon_uvd_fini(rdev);  in cik_fini()
9005 	radeon_vce_fini(rdev);  in cik_fini()
9006 	cik_pcie_gart_fini(rdev);  in cik_fini()
9007 	r600_vram_scratch_fini(rdev);  in cik_fini()
9008 	radeon_gem_fini(rdev);  in cik_fini()
9009 	radeon_fence_driver_fini(rdev);  in cik_fini()
9010 	radeon_bo_fini(rdev);  in cik_fini()
9011 	radeon_atombios_fini(rdev);  in cik_fini()
9012 	kfree(rdev->bios);  in cik_fini()
9013 	rdev->bios = NULL;  in cik_fini()
9019 	struct radeon_device *rdev = dev->dev_private;  in dce8_program_fmt()  local
9093 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,  in dce8_line_buffer_adjust()  argument
9116 			buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;  in dce8_line_buffer_adjust()
9120 			buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;  in dce8_line_buffer_adjust()
9132 	for (i = 0; i < rdev->usec_timeout; i++) {  in dce8_line_buffer_adjust()
9164 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)  in cik_get_number_of_dram_channels()  argument
9535 static void dce8_program_watermarks(struct radeon_device *rdev,  in dce8_program_watermarks()  argument
9551 		if ((rdev->pm.pm_method == PM_METHOD_DPM) &&  in dce8_program_watermarks()
9552 		    rdev->pm.dpm_enabled) {  in dce8_program_watermarks()
9554 				radeon_dpm_get_mclk(rdev, false) * 10;  in dce8_program_watermarks()
9556 				radeon_dpm_get_sclk(rdev, false) * 10;  in dce8_program_watermarks()
9558 			wm_high.yclk = rdev->pm.current_mclk * 10;  in dce8_program_watermarks()
9559 			wm_high.sclk = rdev->pm.current_sclk * 10;  in dce8_program_watermarks()
9575 		wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);  in dce8_program_watermarks()
9586 		    (rdev->disp_priority == 2)) {  in dce8_program_watermarks()
9591 		if ((rdev->pm.pm_method == PM_METHOD_DPM) &&  in dce8_program_watermarks()
9592 		    rdev->pm.dpm_enabled) {  in dce8_program_watermarks()
9594 				radeon_dpm_get_mclk(rdev, true) * 10;  in dce8_program_watermarks()
9596 				radeon_dpm_get_sclk(rdev, true) * 10;  in dce8_program_watermarks()
9598 			wm_low.yclk = rdev->pm.current_mclk * 10;  in dce8_program_watermarks()
9599 			wm_low.sclk = rdev->pm.current_sclk * 10;  in dce8_program_watermarks()
9615 		wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);  in dce8_program_watermarks()
9626 		    (rdev->disp_priority == 2)) {  in dce8_program_watermarks()
9668 void dce8_bandwidth_update(struct radeon_device *rdev)  in dce8_bandwidth_update()  argument
9674 	if (!rdev->mode_info.mode_config_initialized)  in dce8_bandwidth_update()
9677 	radeon_update_display_priority(rdev);  in dce8_bandwidth_update()
9679 	for (i = 0; i < rdev->num_crtc; i++) {  in dce8_bandwidth_update()
9680 		if (rdev->mode_info.crtcs[i]->base.enabled)  in dce8_bandwidth_update()
9683 	for (i = 0; i < rdev->num_crtc; i++) {  in dce8_bandwidth_update()
9684 		mode = &rdev->mode_info.crtcs[i]->base.mode;  in dce8_bandwidth_update()
9685 		lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);  in dce8_bandwidth_update()
9686 		dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);  in dce8_bandwidth_update()
9698 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)  in cik_get_gpu_clock_counter()  argument
9702 	mutex_lock(&rdev->gpu_clock_mutex);  in cik_get_gpu_clock_counter()
9706 	mutex_unlock(&rdev->gpu_clock_mutex);  in cik_get_gpu_clock_counter()
9710 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,  in cik_set_uvd_clock()  argument
9717 	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,  in cik_set_uvd_clock()
9738 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)  in cik_set_uvd_clocks()  argument
9742 	r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);  in cik_set_uvd_clocks()
9746 	r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);  in cik_set_uvd_clocks()
9750 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)  in cik_set_vce_clocks()  argument
9756 	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,  in cik_set_vce_clocks()
9785 static void cik_pcie_gen3_enable(struct radeon_device *rdev)  in cik_pcie_gen3_enable()  argument
9787 	struct pci_dev *root = rdev->pdev->bus->self;  in cik_pcie_gen3_enable()
9793 	if (pci_is_root_bus(rdev->pdev->bus))  in cik_pcie_gen3_enable()
9799 	if (rdev->flags & RADEON_IS_IGP)  in cik_pcie_gen3_enable()
9802 	if (!(rdev->flags & RADEON_IS_PCIE))  in cik_pcie_gen3_enable()
9805 	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);  in cik_pcie_gen3_enable()
9833 	gpu_pos = pci_pcie_cap(rdev->pdev);  in cik_pcie_gen3_enable()
9845 			pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);  in cik_pcie_gen3_enable()
9851 			pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);  in cik_pcie_gen3_enable()
9869 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);  in cik_pcie_gen3_enable()
9874 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);  in cik_pcie_gen3_enable()
9877 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);  in cik_pcie_gen3_enable()
9895 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);  in cik_pcie_gen3_enable()
9898 				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);  in cik_pcie_gen3_enable()
9906 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);  in cik_pcie_gen3_enable()
9909 				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);  in cik_pcie_gen3_enable()
9923 	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);  in cik_pcie_gen3_enable()
9931 	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);  in cik_pcie_gen3_enable()
9937 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_pcie_gen3_enable()
9945 static void cik_program_aspm(struct radeon_device *rdev)  in cik_program_aspm()  argument
9955 	if (rdev->flags & RADEON_IS_IGP)  in cik_program_aspm()
9958 	if (!(rdev->flags & RADEON_IS_PCIE))  in cik_program_aspm()
10023 			    !pci_is_root_bus(rdev->pdev->bus)) {  in cik_program_aspm()
10024 				struct pci_dev *root = rdev->pdev->bus->self;  in cik_program_aspm()