Lines Matching refs:rdev
119 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
120 extern void r600_ih_ring_fini(struct radeon_device *rdev);
121 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
123 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
124 extern void sumo_rlc_fini(struct radeon_device *rdev);
125 extern int sumo_rlc_init(struct radeon_device *rdev);
126 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
127 extern void si_rlc_reset(struct radeon_device *rdev);
128 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
129 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
130 extern int cik_sdma_resume(struct radeon_device *rdev);
131 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
132 extern void cik_sdma_fini(struct radeon_device *rdev);
133 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
134 static void cik_rlc_stop(struct radeon_device *rdev);
135 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
136 static void cik_program_aspm(struct radeon_device *rdev);
137 static void cik_init_pg(struct radeon_device *rdev);
138 static void cik_init_cg(struct radeon_device *rdev);
139 static void cik_fini_pg(struct radeon_device *rdev);
140 static void cik_fini_cg(struct radeon_device *rdev);
141 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
154 int cik_get_allowed_info_register(struct radeon_device *rdev, in cik_get_allowed_info_register() argument
178 int ci_get_temp(struct radeon_device *rdev) in ci_get_temp() argument
197 int kv_get_temp(struct radeon_device *rdev) in kv_get_temp() argument
217 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) in cik_pciep_rreg() argument
222 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); in cik_pciep_rreg()
226 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); in cik_pciep_rreg()
230 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) in cik_pciep_wreg() argument
234 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); in cik_pciep_wreg()
239 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); in cik_pciep_wreg()
1599 static void cik_init_golden_registers(struct radeon_device *rdev) in cik_init_golden_registers() argument
1602 mutex_lock(&rdev->grbm_idx_mutex); in cik_init_golden_registers()
1603 switch (rdev->family) { in cik_init_golden_registers()
1605 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1608 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1611 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1614 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1619 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1622 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1625 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1628 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1633 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1636 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1639 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1642 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1647 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1650 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1653 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1656 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1661 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1664 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1667 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1670 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1677 mutex_unlock(&rdev->grbm_idx_mutex); in cik_init_golden_registers()
1688 u32 cik_get_xclk(struct radeon_device *rdev) in cik_get_xclk() argument
1690 u32 reference_clock = rdev->clock.spll.reference_freq; in cik_get_xclk()
1692 if (rdev->flags & RADEON_IS_IGP) { in cik_get_xclk()
1711 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) in cik_mm_rdoorbell() argument
1713 if (index < rdev->doorbell.num_doorbells) { in cik_mm_rdoorbell()
1714 return readl(rdev->doorbell.ptr + index); in cik_mm_rdoorbell()
1731 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) in cik_mm_wdoorbell() argument
1733 if (index < rdev->doorbell.num_doorbells) { in cik_mm_wdoorbell()
1734 writel(v, rdev->doorbell.ptr + index); in cik_mm_wdoorbell()
1824 static void cik_srbm_select(struct radeon_device *rdev, in cik_srbm_select() argument
1843 int ci_mc_load_microcode(struct radeon_device *rdev) in ci_mc_load_microcode() argument
1852 if (!rdev->mc_fw) in ci_mc_load_microcode()
1855 if (rdev->new_fw) { in ci_mc_load_microcode()
1857 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; in ci_mc_load_microcode()
1863 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); in ci_mc_load_microcode()
1866 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in ci_mc_load_microcode()
1868 ucode_size = rdev->mc_fw->size / 4; in ci_mc_load_microcode()
1870 switch (rdev->family) { in ci_mc_load_microcode()
1882 fw_data = (const __be32 *)rdev->mc_fw->data; in ci_mc_load_microcode()
1899 if (rdev->new_fw) { in ci_mc_load_microcode()
1909 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) { in ci_mc_load_microcode()
1918 if (rdev->new_fw) in ci_mc_load_microcode()
1930 for (i = 0; i < rdev->usec_timeout; i++) { in ci_mc_load_microcode()
1935 for (i = 0; i < rdev->usec_timeout; i++) { in ci_mc_load_microcode()
1957 static int cik_init_microcode(struct radeon_device *rdev) in cik_init_microcode() argument
1971 switch (rdev->family) { in cik_init_microcode()
2039 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); in cik_init_microcode()
2042 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); in cik_init_microcode()
2045 if (rdev->pfp_fw->size != pfp_req_size) { in cik_init_microcode()
2048 rdev->pfp_fw->size, fw_name); in cik_init_microcode()
2053 err = radeon_ucode_validate(rdev->pfp_fw); in cik_init_microcode()
2065 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); in cik_init_microcode()
2068 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); in cik_init_microcode()
2071 if (rdev->me_fw->size != me_req_size) { in cik_init_microcode()
2074 rdev->me_fw->size, fw_name); in cik_init_microcode()
2078 err = radeon_ucode_validate(rdev->me_fw); in cik_init_microcode()
2090 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); in cik_init_microcode()
2093 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); in cik_init_microcode()
2096 if (rdev->ce_fw->size != ce_req_size) { in cik_init_microcode()
2099 rdev->ce_fw->size, fw_name); in cik_init_microcode()
2103 err = radeon_ucode_validate(rdev->ce_fw); in cik_init_microcode()
2115 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); in cik_init_microcode()
2118 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); in cik_init_microcode()
2121 if (rdev->mec_fw->size != mec_req_size) { in cik_init_microcode()
2124 rdev->mec_fw->size, fw_name); in cik_init_microcode()
2128 err = radeon_ucode_validate(rdev->mec_fw); in cik_init_microcode()
2139 if (rdev->family == CHIP_KAVERI) { in cik_init_microcode()
2141 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev); in cik_init_microcode()
2145 err = radeon_ucode_validate(rdev->mec2_fw); in cik_init_microcode()
2155 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); in cik_init_microcode()
2158 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); in cik_init_microcode()
2161 if (rdev->rlc_fw->size != rlc_req_size) { in cik_init_microcode()
2164 rdev->rlc_fw->size, fw_name); in cik_init_microcode()
2168 err = radeon_ucode_validate(rdev->rlc_fw); in cik_init_microcode()
2180 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); in cik_init_microcode()
2183 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); in cik_init_microcode()
2186 if (rdev->sdma_fw->size != sdma_req_size) { in cik_init_microcode()
2189 rdev->sdma_fw->size, fw_name); in cik_init_microcode()
2193 err = radeon_ucode_validate(rdev->sdma_fw); in cik_init_microcode()
2205 if (!(rdev->flags & RADEON_IS_IGP)) { in cik_init_microcode()
2207 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); in cik_init_microcode()
2210 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); in cik_init_microcode()
2213 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); in cik_init_microcode()
2217 if ((rdev->mc_fw->size != mc_req_size) && in cik_init_microcode()
2218 (rdev->mc_fw->size != mc2_req_size)){ in cik_init_microcode()
2221 rdev->mc_fw->size, fw_name); in cik_init_microcode()
2224 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); in cik_init_microcode()
2226 err = radeon_ucode_validate(rdev->mc_fw); in cik_init_microcode()
2238 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); in cik_init_microcode()
2241 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); in cik_init_microcode()
2246 release_firmware(rdev->smc_fw); in cik_init_microcode()
2247 rdev->smc_fw = NULL; in cik_init_microcode()
2249 } else if (rdev->smc_fw->size != smc_req_size) { in cik_init_microcode()
2252 rdev->smc_fw->size, fw_name); in cik_init_microcode()
2256 err = radeon_ucode_validate(rdev->smc_fw); in cik_init_microcode()
2269 rdev->new_fw = false; in cik_init_microcode()
2274 rdev->new_fw = true; in cik_init_microcode()
2283 release_firmware(rdev->pfp_fw); in cik_init_microcode()
2284 rdev->pfp_fw = NULL; in cik_init_microcode()
2285 release_firmware(rdev->me_fw); in cik_init_microcode()
2286 rdev->me_fw = NULL; in cik_init_microcode()
2287 release_firmware(rdev->ce_fw); in cik_init_microcode()
2288 rdev->ce_fw = NULL; in cik_init_microcode()
2289 release_firmware(rdev->mec_fw); in cik_init_microcode()
2290 rdev->mec_fw = NULL; in cik_init_microcode()
2291 release_firmware(rdev->mec2_fw); in cik_init_microcode()
2292 rdev->mec2_fw = NULL; in cik_init_microcode()
2293 release_firmware(rdev->rlc_fw); in cik_init_microcode()
2294 rdev->rlc_fw = NULL; in cik_init_microcode()
2295 release_firmware(rdev->sdma_fw); in cik_init_microcode()
2296 rdev->sdma_fw = NULL; in cik_init_microcode()
2297 release_firmware(rdev->mc_fw); in cik_init_microcode()
2298 rdev->mc_fw = NULL; in cik_init_microcode()
2299 release_firmware(rdev->smc_fw); in cik_init_microcode()
2300 rdev->smc_fw = NULL; in cik_init_microcode()
2319 static void cik_tiling_mode_table_init(struct radeon_device *rdev) in cik_tiling_mode_table_init() argument
2325 u32 num_rbs = rdev->config.cik.max_backends_per_se * in cik_tiling_mode_table_init()
2326 rdev->config.cik.max_shader_engines; in cik_tiling_mode_table_init()
2328 switch (rdev->config.cik.mem_row_size_in_kb) { in cik_tiling_mode_table_init()
2341 num_pipe_configs = rdev->config.cik.max_tile_pipes; in cik_tiling_mode_table_init()
2472 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; in cik_tiling_mode_table_init()
2565 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; in cik_tiling_mode_table_init()
2695 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; in cik_tiling_mode_table_init()
2788 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; in cik_tiling_mode_table_init()
2919 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; in cik_tiling_mode_table_init()
3049 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; in cik_tiling_mode_table_init()
3143 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; in cik_tiling_mode_table_init()
3273 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden; in cik_tiling_mode_table_init()
3366 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden; in cik_tiling_mode_table_init()
3384 static void cik_select_se_sh(struct radeon_device *rdev, in cik_select_se_sh() argument
3430 static u32 cik_get_rb_disabled(struct radeon_device *rdev, in cik_get_rb_disabled() argument
3460 static void cik_setup_rb(struct radeon_device *rdev, in cik_setup_rb() argument
3469 mutex_lock(&rdev->grbm_idx_mutex); in cik_setup_rb()
3472 cik_select_se_sh(rdev, i, j); in cik_setup_rb()
3473 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); in cik_setup_rb()
3474 if (rdev->family == CHIP_HAWAII) in cik_setup_rb()
3480 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_setup_rb()
3481 mutex_unlock(&rdev->grbm_idx_mutex); in cik_setup_rb()
3490 rdev->config.cik.backend_enable_mask = enabled_rbs; in cik_setup_rb()
3492 mutex_lock(&rdev->grbm_idx_mutex); in cik_setup_rb()
3494 cik_select_se_sh(rdev, i, 0xffffffff); in cik_setup_rb()
3519 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_setup_rb()
3520 mutex_unlock(&rdev->grbm_idx_mutex); in cik_setup_rb()
3531 static void cik_gpu_init(struct radeon_device *rdev) in cik_gpu_init() argument
3539 switch (rdev->family) { in cik_gpu_init()
3541 rdev->config.cik.max_shader_engines = 2; in cik_gpu_init()
3542 rdev->config.cik.max_tile_pipes = 4; in cik_gpu_init()
3543 rdev->config.cik.max_cu_per_sh = 7; in cik_gpu_init()
3544 rdev->config.cik.max_sh_per_se = 1; in cik_gpu_init()
3545 rdev->config.cik.max_backends_per_se = 2; in cik_gpu_init()
3546 rdev->config.cik.max_texture_channel_caches = 4; in cik_gpu_init()
3547 rdev->config.cik.max_gprs = 256; in cik_gpu_init()
3548 rdev->config.cik.max_gs_threads = 32; in cik_gpu_init()
3549 rdev->config.cik.max_hw_contexts = 8; in cik_gpu_init()
3551 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; in cik_gpu_init()
3552 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; in cik_gpu_init()
3553 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; in cik_gpu_init()
3554 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; in cik_gpu_init()
3558 rdev->config.cik.max_shader_engines = 4; in cik_gpu_init()
3559 rdev->config.cik.max_tile_pipes = 16; in cik_gpu_init()
3560 rdev->config.cik.max_cu_per_sh = 11; in cik_gpu_init()
3561 rdev->config.cik.max_sh_per_se = 1; in cik_gpu_init()
3562 rdev->config.cik.max_backends_per_se = 4; in cik_gpu_init()
3563 rdev->config.cik.max_texture_channel_caches = 16; in cik_gpu_init()
3564 rdev->config.cik.max_gprs = 256; in cik_gpu_init()
3565 rdev->config.cik.max_gs_threads = 32; in cik_gpu_init()
3566 rdev->config.cik.max_hw_contexts = 8; in cik_gpu_init()
3568 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; in cik_gpu_init()
3569 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; in cik_gpu_init()
3570 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; in cik_gpu_init()
3571 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; in cik_gpu_init()
3575 rdev->config.cik.max_shader_engines = 1; in cik_gpu_init()
3576 rdev->config.cik.max_tile_pipes = 4; in cik_gpu_init()
3577 if ((rdev->pdev->device == 0x1304) || in cik_gpu_init()
3578 (rdev->pdev->device == 0x1305) || in cik_gpu_init()
3579 (rdev->pdev->device == 0x130C) || in cik_gpu_init()
3580 (rdev->pdev->device == 0x130F) || in cik_gpu_init()
3581 (rdev->pdev->device == 0x1310) || in cik_gpu_init()
3582 (rdev->pdev->device == 0x1311) || in cik_gpu_init()
3583 (rdev->pdev->device == 0x131C)) { in cik_gpu_init()
3584 rdev->config.cik.max_cu_per_sh = 8; in cik_gpu_init()
3585 rdev->config.cik.max_backends_per_se = 2; in cik_gpu_init()
3586 } else if ((rdev->pdev->device == 0x1309) || in cik_gpu_init()
3587 (rdev->pdev->device == 0x130A) || in cik_gpu_init()
3588 (rdev->pdev->device == 0x130D) || in cik_gpu_init()
3589 (rdev->pdev->device == 0x1313) || in cik_gpu_init()
3590 (rdev->pdev->device == 0x131D)) { in cik_gpu_init()
3591 rdev->config.cik.max_cu_per_sh = 6; in cik_gpu_init()
3592 rdev->config.cik.max_backends_per_se = 2; in cik_gpu_init()
3593 } else if ((rdev->pdev->device == 0x1306) || in cik_gpu_init()
3594 (rdev->pdev->device == 0x1307) || in cik_gpu_init()
3595 (rdev->pdev->device == 0x130B) || in cik_gpu_init()
3596 (rdev->pdev->device == 0x130E) || in cik_gpu_init()
3597 (rdev->pdev->device == 0x1315) || in cik_gpu_init()
3598 (rdev->pdev->device == 0x1318) || in cik_gpu_init()
3599 (rdev->pdev->device == 0x131B)) { in cik_gpu_init()
3600 rdev->config.cik.max_cu_per_sh = 4; in cik_gpu_init()
3601 rdev->config.cik.max_backends_per_se = 1; in cik_gpu_init()
3603 rdev->config.cik.max_cu_per_sh = 3; in cik_gpu_init()
3604 rdev->config.cik.max_backends_per_se = 1; in cik_gpu_init()
3606 rdev->config.cik.max_sh_per_se = 1; in cik_gpu_init()
3607 rdev->config.cik.max_texture_channel_caches = 4; in cik_gpu_init()
3608 rdev->config.cik.max_gprs = 256; in cik_gpu_init()
3609 rdev->config.cik.max_gs_threads = 16; in cik_gpu_init()
3610 rdev->config.cik.max_hw_contexts = 8; in cik_gpu_init()
3612 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; in cik_gpu_init()
3613 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; in cik_gpu_init()
3614 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; in cik_gpu_init()
3615 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; in cik_gpu_init()
3621 rdev->config.cik.max_shader_engines = 1; in cik_gpu_init()
3622 rdev->config.cik.max_tile_pipes = 2; in cik_gpu_init()
3623 rdev->config.cik.max_cu_per_sh = 2; in cik_gpu_init()
3624 rdev->config.cik.max_sh_per_se = 1; in cik_gpu_init()
3625 rdev->config.cik.max_backends_per_se = 1; in cik_gpu_init()
3626 rdev->config.cik.max_texture_channel_caches = 2; in cik_gpu_init()
3627 rdev->config.cik.max_gprs = 256; in cik_gpu_init()
3628 rdev->config.cik.max_gs_threads = 16; in cik_gpu_init()
3629 rdev->config.cik.max_hw_contexts = 8; in cik_gpu_init()
3631 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; in cik_gpu_init()
3632 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; in cik_gpu_init()
3633 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; in cik_gpu_init()
3634 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; in cik_gpu_init()
3657 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; in cik_gpu_init()
3658 rdev->config.cik.mem_max_burst_length_bytes = 256; in cik_gpu_init()
3660 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; in cik_gpu_init()
3661 if (rdev->config.cik.mem_row_size_in_kb > 4) in cik_gpu_init()
3662 rdev->config.cik.mem_row_size_in_kb = 4; in cik_gpu_init()
3664 rdev->config.cik.shader_engine_tile_size = 32; in cik_gpu_init()
3665 rdev->config.cik.num_gpus = 1; in cik_gpu_init()
3666 rdev->config.cik.multi_gpu_tile_size = 64; in cik_gpu_init()
3670 switch (rdev->config.cik.mem_row_size_in_kb) { in cik_gpu_init()
3690 rdev->config.cik.tile_config = 0; in cik_gpu_init()
3691 switch (rdev->config.cik.num_tile_pipes) { in cik_gpu_init()
3693 rdev->config.cik.tile_config |= (0 << 0); in cik_gpu_init()
3696 rdev->config.cik.tile_config |= (1 << 0); in cik_gpu_init()
3699 rdev->config.cik.tile_config |= (2 << 0); in cik_gpu_init()
3704 rdev->config.cik.tile_config |= (3 << 0); in cik_gpu_init()
3707 rdev->config.cik.tile_config |= in cik_gpu_init()
3709 rdev->config.cik.tile_config |= in cik_gpu_init()
3711 rdev->config.cik.tile_config |= in cik_gpu_init()
3723 cik_tiling_mode_table_init(rdev); in cik_gpu_init()
3725 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, in cik_gpu_init()
3726 rdev->config.cik.max_sh_per_se, in cik_gpu_init()
3727 rdev->config.cik.max_backends_per_se); in cik_gpu_init()
3729 rdev->config.cik.active_cus = 0; in cik_gpu_init()
3730 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { in cik_gpu_init()
3731 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { in cik_gpu_init()
3732 rdev->config.cik.active_cus += in cik_gpu_init()
3733 hweight32(cik_get_cu_active_bitmap(rdev, i, j)); in cik_gpu_init()
3740 mutex_lock(&rdev->grbm_idx_mutex); in cik_gpu_init()
3745 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_gpu_init()
3772 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | in cik_gpu_init()
3773 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | in cik_gpu_init()
3774 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | in cik_gpu_init()
3775 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); in cik_gpu_init()
3801 mutex_unlock(&rdev->grbm_idx_mutex); in cik_gpu_init()
3819 static void cik_scratch_init(struct radeon_device *rdev) in cik_scratch_init() argument
3823 rdev->scratch.num_reg = 7; in cik_scratch_init()
3824 rdev->scratch.reg_base = SCRATCH_REG0; in cik_scratch_init()
3825 for (i = 0; i < rdev->scratch.num_reg; i++) { in cik_scratch_init()
3826 rdev->scratch.free[i] = true; in cik_scratch_init()
3827 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); in cik_scratch_init()
3842 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) in cik_ring_test() argument
3849 r = radeon_scratch_get(rdev, &scratch); in cik_ring_test()
3855 r = radeon_ring_lock(rdev, ring, 3); in cik_ring_test()
3858 radeon_scratch_free(rdev, scratch); in cik_ring_test()
3864 radeon_ring_unlock_commit(rdev, ring, false); in cik_ring_test()
3866 for (i = 0; i < rdev->usec_timeout; i++) { in cik_ring_test()
3872 if (i < rdev->usec_timeout) { in cik_ring_test()
3879 radeon_scratch_free(rdev, scratch); in cik_ring_test()
3891 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, in cik_hdp_flush_cp_ring_emit() argument
3894 struct radeon_ring *ring = &rdev->ring[ridx]; in cik_hdp_flush_cp_ring_emit()
3937 void cik_fence_gfx_ring_emit(struct radeon_device *rdev, in cik_fence_gfx_ring_emit() argument
3940 struct radeon_ring *ring = &rdev->ring[fence->ring]; in cik_fence_gfx_ring_emit()
3941 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; in cik_fence_gfx_ring_emit()
3978 void cik_fence_compute_ring_emit(struct radeon_device *rdev, in cik_fence_compute_ring_emit() argument
3981 struct radeon_ring *ring = &rdev->ring[fence->ring]; in cik_fence_compute_ring_emit()
3982 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; in cik_fence_compute_ring_emit()
4008 bool cik_semaphore_ring_emit(struct radeon_device *rdev, in cik_semaphore_ring_emit() argument
4042 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, in cik_copy_cpdma() argument
4049 int ring_index = rdev->asic->copy.blit_ring_index; in cik_copy_cpdma()
4050 struct radeon_ring *ring = &rdev->ring[ring_index]; in cik_copy_cpdma()
4059 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); in cik_copy_cpdma()
4062 radeon_sync_free(rdev, &sync, NULL); in cik_copy_cpdma()
4066 radeon_sync_resv(rdev, &sync, resv, false); in cik_copy_cpdma()
4067 radeon_sync_rings(rdev, &sync, ring->idx); in cik_copy_cpdma()
4088 r = radeon_fence_emit(rdev, &fence, ring->idx); in cik_copy_cpdma()
4090 radeon_ring_unlock_undo(rdev, ring); in cik_copy_cpdma()
4091 radeon_sync_free(rdev, &sync, NULL); in cik_copy_cpdma()
4095 radeon_ring_unlock_commit(rdev, ring, false); in cik_copy_cpdma()
4096 radeon_sync_free(rdev, &sync, fence); in cik_copy_cpdma()
4116 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) in cik_ring_ib_execute() argument
4118 struct radeon_ring *ring = &rdev->ring[ib->ring]; in cik_ring_ib_execute()
4136 } else if (rdev->wb.enabled) { in cik_ring_ib_execute()
4170 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) in cik_ib_test() argument
4178 r = radeon_scratch_get(rdev, &scratch); in cik_ib_test()
4184 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); in cik_ib_test()
4187 radeon_scratch_free(rdev, scratch); in cik_ib_test()
4194 r = radeon_ib_schedule(rdev, &ib, NULL, false); in cik_ib_test()
4196 radeon_scratch_free(rdev, scratch); in cik_ib_test()
4197 radeon_ib_free(rdev, &ib); in cik_ib_test()
4204 radeon_scratch_free(rdev, scratch); in cik_ib_test()
4205 radeon_ib_free(rdev, &ib); in cik_ib_test()
4208 for (i = 0; i < rdev->usec_timeout; i++) { in cik_ib_test()
4214 if (i < rdev->usec_timeout) { in cik_ib_test()
4221 radeon_scratch_free(rdev, scratch); in cik_ib_test()
4222 radeon_ib_free(rdev, &ib); in cik_ib_test()
4257 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) in cik_cp_gfx_enable() argument
4262 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) in cik_cp_gfx_enable()
4263 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); in cik_cp_gfx_enable()
4265 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; in cik_cp_gfx_enable()
4278 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) in cik_cp_gfx_load_microcode() argument
4282 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) in cik_cp_gfx_load_microcode()
4285 cik_cp_gfx_enable(rdev, false); in cik_cp_gfx_load_microcode()
4287 if (rdev->new_fw) { in cik_cp_gfx_load_microcode()
4289 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; in cik_cp_gfx_load_microcode()
4291 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; in cik_cp_gfx_load_microcode()
4293 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; in cik_cp_gfx_load_microcode()
4303 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); in cik_cp_gfx_load_microcode()
4312 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); in cik_cp_gfx_load_microcode()
4321 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); in cik_cp_gfx_load_microcode()
4332 fw_data = (const __be32 *)rdev->pfp_fw->data; in cik_cp_gfx_load_microcode()
4339 fw_data = (const __be32 *)rdev->ce_fw->data; in cik_cp_gfx_load_microcode()
4346 fw_data = (const __be32 *)rdev->me_fw->data; in cik_cp_gfx_load_microcode()
4365 static int cik_cp_gfx_start(struct radeon_device *rdev) in cik_cp_gfx_start() argument
4367 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in cik_cp_gfx_start()
4371 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); in cik_cp_gfx_start()
4375 cik_cp_gfx_enable(rdev, true); in cik_cp_gfx_start()
4377 r = radeon_ring_lock(rdev, ring, cik_default_size + 17); in cik_cp_gfx_start()
4412 radeon_ring_unlock_commit(rdev, ring, false); in cik_cp_gfx_start()
4425 static void cik_cp_gfx_fini(struct radeon_device *rdev) in cik_cp_gfx_fini() argument
4427 cik_cp_gfx_enable(rdev, false); in cik_cp_gfx_fini()
4428 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); in cik_cp_gfx_fini()
4440 static int cik_cp_gfx_resume(struct radeon_device *rdev) in cik_cp_gfx_resume() argument
4449 if (rdev->family != CHIP_HAWAII) in cik_cp_gfx_resume()
4458 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); in cik_cp_gfx_resume()
4462 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in cik_cp_gfx_resume()
4476 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); in cik_cp_gfx_resume()
4477 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); in cik_cp_gfx_resume()
4482 if (!rdev->wb.enabled) in cik_cp_gfx_resume()
4493 cik_cp_gfx_start(rdev); in cik_cp_gfx_resume()
4494 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; in cik_cp_gfx_resume()
4495 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); in cik_cp_gfx_resume()
4497 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; in cik_cp_gfx_resume()
4501 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) in cik_cp_gfx_resume()
4502 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); in cik_cp_gfx_resume()
4507 u32 cik_gfx_get_rptr(struct radeon_device *rdev, in cik_gfx_get_rptr() argument
4512 if (rdev->wb.enabled) in cik_gfx_get_rptr()
4513 rptr = rdev->wb.wb[ring->rptr_offs/4]; in cik_gfx_get_rptr()
4520 u32 cik_gfx_get_wptr(struct radeon_device *rdev, in cik_gfx_get_wptr() argument
4530 void cik_gfx_set_wptr(struct radeon_device *rdev, in cik_gfx_set_wptr() argument
4537 u32 cik_compute_get_rptr(struct radeon_device *rdev, in cik_compute_get_rptr() argument
4542 if (rdev->wb.enabled) { in cik_compute_get_rptr()
4543 rptr = rdev->wb.wb[ring->rptr_offs/4]; in cik_compute_get_rptr()
4545 mutex_lock(&rdev->srbm_mutex); in cik_compute_get_rptr()
4546 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); in cik_compute_get_rptr()
4548 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_compute_get_rptr()
4549 mutex_unlock(&rdev->srbm_mutex); in cik_compute_get_rptr()
4555 u32 cik_compute_get_wptr(struct radeon_device *rdev, in cik_compute_get_wptr() argument
4560 if (rdev->wb.enabled) { in cik_compute_get_wptr()
4562 wptr = rdev->wb.wb[ring->wptr_offs/4]; in cik_compute_get_wptr()
4564 mutex_lock(&rdev->srbm_mutex); in cik_compute_get_wptr()
4565 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); in cik_compute_get_wptr()
4567 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_compute_get_wptr()
4568 mutex_unlock(&rdev->srbm_mutex); in cik_compute_get_wptr()
4574 void cik_compute_set_wptr(struct radeon_device *rdev, in cik_compute_set_wptr() argument
4578 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; in cik_compute_set_wptr()
4582 static void cik_compute_stop(struct radeon_device *rdev, in cik_compute_stop() argument
4587 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); in cik_compute_stop()
4595 for (j = 0; j < rdev->usec_timeout; j++) { in cik_compute_stop()
4604 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_compute_stop()
4615 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) in cik_cp_compute_enable() argument
4624 mutex_lock(&rdev->srbm_mutex); in cik_cp_compute_enable()
4625 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); in cik_cp_compute_enable()
4626 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); in cik_cp_compute_enable()
4627 mutex_unlock(&rdev->srbm_mutex); in cik_cp_compute_enable()
4630 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; in cik_cp_compute_enable()
4631 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; in cik_cp_compute_enable()
4644 static int cik_cp_compute_load_microcode(struct radeon_device *rdev) in cik_cp_compute_load_microcode() argument
4648 if (!rdev->mec_fw) in cik_cp_compute_load_microcode()
4651 cik_cp_compute_enable(rdev, false); in cik_cp_compute_load_microcode()
4653 if (rdev->new_fw) { in cik_cp_compute_load_microcode()
4655 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; in cik_cp_compute_load_microcode()
4663 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); in cik_cp_compute_load_microcode()
4671 if (rdev->family == CHIP_KAVERI) { in cik_cp_compute_load_microcode()
4673 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; in cik_cp_compute_load_microcode()
4676 (rdev->mec2_fw->data + in cik_cp_compute_load_microcode()
4688 fw_data = (const __be32 *)rdev->mec_fw->data; in cik_cp_compute_load_microcode()
4694 if (rdev->family == CHIP_KAVERI) { in cik_cp_compute_load_microcode()
4696 fw_data = (const __be32 *)rdev->mec_fw->data; in cik_cp_compute_load_microcode()
4715 static int cik_cp_compute_start(struct radeon_device *rdev) in cik_cp_compute_start() argument
4717 cik_cp_compute_enable(rdev, true); in cik_cp_compute_start()
4730 static void cik_cp_compute_fini(struct radeon_device *rdev) in cik_cp_compute_fini() argument
4734 cik_cp_compute_enable(rdev, false); in cik_cp_compute_fini()
4742 if (rdev->ring[idx].mqd_obj) { in cik_cp_compute_fini()
4743 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); in cik_cp_compute_fini()
4745 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); in cik_cp_compute_fini()
4747 radeon_bo_unpin(rdev->ring[idx].mqd_obj); in cik_cp_compute_fini()
4748 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); in cik_cp_compute_fini()
4750 radeon_bo_unref(&rdev->ring[idx].mqd_obj); in cik_cp_compute_fini()
4751 rdev->ring[idx].mqd_obj = NULL; in cik_cp_compute_fini()
4756 static void cik_mec_fini(struct radeon_device *rdev) in cik_mec_fini() argument
4760 if (rdev->mec.hpd_eop_obj) { in cik_mec_fini()
4761 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); in cik_mec_fini()
4763 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); in cik_mec_fini()
4764 radeon_bo_unpin(rdev->mec.hpd_eop_obj); in cik_mec_fini()
4765 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); in cik_mec_fini()
4767 radeon_bo_unref(&rdev->mec.hpd_eop_obj); in cik_mec_fini()
4768 rdev->mec.hpd_eop_obj = NULL; in cik_mec_fini()
4774 static int cik_mec_init(struct radeon_device *rdev) in cik_mec_init() argument
4785 rdev->mec.num_mec = 1; in cik_mec_init()
4786 rdev->mec.num_pipe = 1; in cik_mec_init()
4787 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; in cik_mec_init()
4789 if (rdev->mec.hpd_eop_obj == NULL) { in cik_mec_init()
4790 r = radeon_bo_create(rdev, in cik_mec_init()
4791 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, in cik_mec_init()
4794 &rdev->mec.hpd_eop_obj); in cik_mec_init()
4796 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); in cik_mec_init()
4801 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); in cik_mec_init()
4803 cik_mec_fini(rdev); in cik_mec_init()
4806 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, in cik_mec_init()
4807 &rdev->mec.hpd_eop_gpu_addr); in cik_mec_init()
4809 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); in cik_mec_init()
4810 cik_mec_fini(rdev); in cik_mec_init()
4813 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); in cik_mec_init()
4815 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); in cik_mec_init()
4816 cik_mec_fini(rdev); in cik_mec_init()
4821 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); in cik_mec_init()
4823 radeon_bo_kunmap(rdev->mec.hpd_eop_obj); in cik_mec_init()
4824 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); in cik_mec_init()
4905 static int cik_cp_compute_resume(struct radeon_device *rdev) in cik_cp_compute_resume() argument
4917 r = cik_cp_compute_start(rdev); in cik_cp_compute_resume()
4927 mutex_lock(&rdev->srbm_mutex); in cik_cp_compute_resume()
4929 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr; in cik_cp_compute_resume()
4931 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_cp_compute_resume()
4946 mutex_unlock(&rdev->srbm_mutex); in cik_cp_compute_resume()
4955 if (rdev->ring[idx].mqd_obj == NULL) { in cik_cp_compute_resume()
4956 r = radeon_bo_create(rdev, in cik_cp_compute_resume()
4960 NULL, &rdev->ring[idx].mqd_obj); in cik_cp_compute_resume()
4962 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); in cik_cp_compute_resume()
4967 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); in cik_cp_compute_resume()
4969 cik_cp_compute_fini(rdev); in cik_cp_compute_resume()
4972 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, in cik_cp_compute_resume()
4975 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); in cik_cp_compute_resume()
4976 cik_cp_compute_fini(rdev); in cik_cp_compute_resume()
4979 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); in cik_cp_compute_resume()
4981 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); in cik_cp_compute_resume()
4982 cik_cp_compute_fini(rdev); in cik_cp_compute_resume()
4996 mutex_lock(&rdev->srbm_mutex); in cik_cp_compute_resume()
4997 cik_srbm_select(rdev, rdev->ring[idx].me, in cik_cp_compute_resume()
4998 rdev->ring[idx].pipe, in cik_cp_compute_resume()
4999 rdev->ring[idx].queue, 0); in cik_cp_compute_resume()
5022 for (j = 0; j < rdev->usec_timeout; j++) { in cik_cp_compute_resume()
5043 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; in cik_cp_compute_resume()
5055 order_base_2(rdev->ring[idx].ring_size / 8); in cik_cp_compute_resume()
5069 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; in cik_cp_compute_resume()
5071 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; in cik_cp_compute_resume()
5080 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; in cik_cp_compute_resume()
5082 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; in cik_cp_compute_resume()
5097 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); in cik_cp_compute_resume()
5109 rdev->ring[idx].wptr = 0; in cik_cp_compute_resume()
5110 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; in cik_cp_compute_resume()
5122 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_cp_compute_resume()
5123 mutex_unlock(&rdev->srbm_mutex); in cik_cp_compute_resume()
5125 radeon_bo_kunmap(rdev->ring[idx].mqd_obj); in cik_cp_compute_resume()
5126 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); in cik_cp_compute_resume()
5128 rdev->ring[idx].ready = true; in cik_cp_compute_resume()
5129 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); in cik_cp_compute_resume()
5131 rdev->ring[idx].ready = false; in cik_cp_compute_resume()
5137 static void cik_cp_enable(struct radeon_device *rdev, bool enable) in cik_cp_enable() argument
5139 cik_cp_gfx_enable(rdev, enable); in cik_cp_enable()
5140 cik_cp_compute_enable(rdev, enable); in cik_cp_enable()
5143 static int cik_cp_load_microcode(struct radeon_device *rdev) in cik_cp_load_microcode() argument
5147 r = cik_cp_gfx_load_microcode(rdev); in cik_cp_load_microcode()
5150 r = cik_cp_compute_load_microcode(rdev); in cik_cp_load_microcode()
5157 static void cik_cp_fini(struct radeon_device *rdev) in cik_cp_fini() argument
5159 cik_cp_gfx_fini(rdev); in cik_cp_fini()
5160 cik_cp_compute_fini(rdev); in cik_cp_fini()
5163 static int cik_cp_resume(struct radeon_device *rdev) in cik_cp_resume() argument
5167 cik_enable_gui_idle_interrupt(rdev, false); in cik_cp_resume()
5169 r = cik_cp_load_microcode(rdev); in cik_cp_resume()
5173 r = cik_cp_gfx_resume(rdev); in cik_cp_resume()
5176 r = cik_cp_compute_resume(rdev); in cik_cp_resume()
5180 cik_enable_gui_idle_interrupt(rdev, true); in cik_cp_resume()
5185 static void cik_print_gpu_status_regs(struct radeon_device *rdev) in cik_print_gpu_status_regs() argument
5187 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", in cik_print_gpu_status_regs()
5189 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", in cik_print_gpu_status_regs()
5191 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", in cik_print_gpu_status_regs()
5193 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", in cik_print_gpu_status_regs()
5195 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", in cik_print_gpu_status_regs()
5197 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", in cik_print_gpu_status_regs()
5199 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", in cik_print_gpu_status_regs()
5201 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", in cik_print_gpu_status_regs()
5203 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", in cik_print_gpu_status_regs()
5205 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", in cik_print_gpu_status_regs()
5207 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); in cik_print_gpu_status_regs()
5208 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", in cik_print_gpu_status_regs()
5210 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", in cik_print_gpu_status_regs()
5212 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", in cik_print_gpu_status_regs()
5214 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", in cik_print_gpu_status_regs()
5216 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", in cik_print_gpu_status_regs()
5218 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); in cik_print_gpu_status_regs()
5219 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); in cik_print_gpu_status_regs()
5220 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", in cik_print_gpu_status_regs()
5222 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); in cik_print_gpu_status_regs()
5234 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) in cik_gpu_check_soft_reset() argument
5294 if (evergreen_is_display_hung(rdev)) in cik_gpu_check_soft_reset()
5314 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) in cik_gpu_soft_reset() argument
5323 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); in cik_gpu_soft_reset()
5325 cik_print_gpu_status_regs(rdev); in cik_gpu_soft_reset()
5326 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", in cik_gpu_soft_reset()
5328 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", in cik_gpu_soft_reset()
5332 cik_fini_pg(rdev); in cik_gpu_soft_reset()
5333 cik_fini_cg(rdev); in cik_gpu_soft_reset()
5336 cik_rlc_stop(rdev); in cik_gpu_soft_reset()
5357 evergreen_mc_stop(rdev, &save); in cik_gpu_soft_reset()
5358 if (evergreen_mc_wait_for_idle(rdev)) { in cik_gpu_soft_reset()
5359 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); in cik_gpu_soft_reset()
5395 if (!(rdev->flags & RADEON_IS_IGP)) { in cik_gpu_soft_reset()
5403 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); in cik_gpu_soft_reset()
5417 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); in cik_gpu_soft_reset()
5431 evergreen_mc_resume(rdev, &save); in cik_gpu_soft_reset()
5434 cik_print_gpu_status_regs(rdev); in cik_gpu_soft_reset()
5443 static void kv_save_regs_for_reset(struct radeon_device *rdev, in kv_save_regs_for_reset() argument
5455 static void kv_restore_regs_for_reset(struct radeon_device *rdev, in kv_restore_regs_for_reset() argument
5528 static void cik_gpu_pci_config_reset(struct radeon_device *rdev) in cik_gpu_pci_config_reset() argument
5534 dev_info(rdev->dev, "GPU pci config reset\n"); in cik_gpu_pci_config_reset()
5539 cik_fini_pg(rdev); in cik_gpu_pci_config_reset()
5540 cik_fini_cg(rdev); in cik_gpu_pci_config_reset()
5559 cik_rlc_stop(rdev); in cik_gpu_pci_config_reset()
5564 evergreen_mc_stop(rdev, &save); in cik_gpu_pci_config_reset()
5565 if (evergreen_mc_wait_for_idle(rdev)) { in cik_gpu_pci_config_reset()
5566 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); in cik_gpu_pci_config_reset()
5569 if (rdev->flags & RADEON_IS_IGP) in cik_gpu_pci_config_reset()
5570 kv_save_regs_for_reset(rdev, &kv_save); in cik_gpu_pci_config_reset()
5573 pci_clear_master(rdev->pdev); in cik_gpu_pci_config_reset()
5575 radeon_pci_config_reset(rdev); in cik_gpu_pci_config_reset()
5580 for (i = 0; i < rdev->usec_timeout; i++) { in cik_gpu_pci_config_reset()
5587 if (rdev->flags & RADEON_IS_IGP) in cik_gpu_pci_config_reset()
5588 kv_restore_regs_for_reset(rdev, &kv_save); in cik_gpu_pci_config_reset()
5600 int cik_asic_reset(struct radeon_device *rdev) in cik_asic_reset() argument
5604 reset_mask = cik_gpu_check_soft_reset(rdev); in cik_asic_reset()
5607 r600_set_bios_scratch_engine_hung(rdev, true); in cik_asic_reset()
5610 cik_gpu_soft_reset(rdev, reset_mask); in cik_asic_reset()
5612 reset_mask = cik_gpu_check_soft_reset(rdev); in cik_asic_reset()
5616 cik_gpu_pci_config_reset(rdev); in cik_asic_reset()
5618 reset_mask = cik_gpu_check_soft_reset(rdev); in cik_asic_reset()
5621 r600_set_bios_scratch_engine_hung(rdev, false); in cik_asic_reset()
5635 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) in cik_gfx_is_lockup() argument
5637 u32 reset_mask = cik_gpu_check_soft_reset(rdev); in cik_gfx_is_lockup()
5642 radeon_ring_lockup_update(rdev, ring); in cik_gfx_is_lockup()
5645 return radeon_ring_test_lockup(rdev, ring); in cik_gfx_is_lockup()
5657 static void cik_mc_program(struct radeon_device *rdev) in cik_mc_program() argument
5673 evergreen_mc_stop(rdev, &save); in cik_mc_program()
5674 if (radeon_mc_wait_for_idle(rdev)) { in cik_mc_program()
5675 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); in cik_mc_program()
5681 rdev->mc.vram_start >> 12); in cik_mc_program()
5683 rdev->mc.vram_end >> 12); in cik_mc_program()
5685 rdev->vram_scratch.gpu_addr >> 12); in cik_mc_program()
5686 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; in cik_mc_program()
5687 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); in cik_mc_program()
5690 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); in cik_mc_program()
5696 if (radeon_mc_wait_for_idle(rdev)) { in cik_mc_program()
5697 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); in cik_mc_program()
5699 evergreen_mc_resume(rdev, &save); in cik_mc_program()
5702 rv515_vga_render_disable(rdev); in cik_mc_program()
5714 static int cik_mc_init(struct radeon_device *rdev) in cik_mc_init() argument
5720 rdev->mc.vram_is_ddr = true; in cik_mc_init()
5758 rdev->mc.vram_width = numchan * chansize; in cik_mc_init()
5760 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); in cik_mc_init()
5761 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); in cik_mc_init()
5763 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; in cik_mc_init()
5764 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; in cik_mc_init()
5765 rdev->mc.visible_vram_size = rdev->mc.aper_size; in cik_mc_init()
5766 si_vram_gtt_location(rdev, &rdev->mc); in cik_mc_init()
5767 radeon_update_bandwidth_info(rdev); in cik_mc_init()
5785 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) in cik_pcie_gart_tlb_flush() argument
5794 static void cik_pcie_init_compute_vmid(struct radeon_device *rdev) in cik_pcie_init_compute_vmid() argument
5803 mutex_lock(&rdev->srbm_mutex); in cik_pcie_init_compute_vmid()
5805 cik_srbm_select(rdev, 0, 0, 0, i); in cik_pcie_init_compute_vmid()
5812 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_pcie_init_compute_vmid()
5813 mutex_unlock(&rdev->srbm_mutex); in cik_pcie_init_compute_vmid()
5827 static int cik_pcie_gart_enable(struct radeon_device *rdev) in cik_pcie_gart_enable() argument
5831 if (rdev->gart.robj == NULL) { in cik_pcie_gart_enable()
5832 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); in cik_pcie_gart_enable()
5835 r = radeon_gart_table_vram_pin(rdev); in cik_pcie_gart_enable()
5858 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); in cik_pcie_gart_enable()
5859 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); in cik_pcie_gart_enable()
5860 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); in cik_pcie_gart_enable()
5862 (u32)(rdev->dummy_page.addr >> 12)); in cik_pcie_gart_enable()
5874 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1); in cik_pcie_gart_enable()
5878 rdev->vm_manager.saved_table_addr[i]); in cik_pcie_gart_enable()
5881 rdev->vm_manager.saved_table_addr[i]); in cik_pcie_gart_enable()
5886 (u32)(rdev->dummy_page.addr >> 12)); in cik_pcie_gart_enable()
5903 if (rdev->family == CHIP_KAVERI) { in cik_pcie_gart_enable()
5911 mutex_lock(&rdev->srbm_mutex); in cik_pcie_gart_enable()
5913 cik_srbm_select(rdev, 0, 0, 0, i); in cik_pcie_gart_enable()
5926 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_pcie_gart_enable()
5927 mutex_unlock(&rdev->srbm_mutex); in cik_pcie_gart_enable()
5929 cik_pcie_init_compute_vmid(rdev); in cik_pcie_gart_enable()
5931 cik_pcie_gart_tlb_flush(rdev); in cik_pcie_gart_enable()
5933 (unsigned)(rdev->mc.gtt_size >> 20), in cik_pcie_gart_enable()
5934 (unsigned long long)rdev->gart.table_addr); in cik_pcie_gart_enable()
5935 rdev->gart.ready = true; in cik_pcie_gart_enable()
5946 static void cik_pcie_gart_disable(struct radeon_device *rdev) in cik_pcie_gart_disable() argument
5956 rdev->vm_manager.saved_table_addr[i] = RREG32(reg); in cik_pcie_gart_disable()
5975 radeon_gart_table_vram_unpin(rdev); in cik_pcie_gart_disable()
5985 static void cik_pcie_gart_fini(struct radeon_device *rdev) in cik_pcie_gart_fini() argument
5987 cik_pcie_gart_disable(rdev); in cik_pcie_gart_fini()
5988 radeon_gart_table_vram_free(rdev); in cik_pcie_gart_fini()
5989 radeon_gart_fini(rdev); in cik_pcie_gart_fini()
6001 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) in cik_ib_parse() argument
6021 int cik_vm_init(struct radeon_device *rdev) in cik_vm_init() argument
6029 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS; in cik_vm_init()
6031 if (rdev->flags & RADEON_IS_IGP) { in cik_vm_init()
6034 rdev->vm_manager.vram_base_offset = tmp; in cik_vm_init()
6036 rdev->vm_manager.vram_base_offset = 0; in cik_vm_init()
6048 void cik_vm_fini(struct radeon_device *rdev) in cik_vm_fini() argument
6061 static void cik_vm_decode_fault(struct radeon_device *rdev, in cik_vm_decode_fault() argument
6070 if (rdev->family == CHIP_HAWAII) in cik_vm_decode_fault()
6089 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, in cik_vm_flush() argument
6134 cik_hdp_flush_cp_ring_emit(rdev, ring->idx); in cik_vm_flush()
6169 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, in cik_enable_gui_idle_interrupt() argument
6181 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) in cik_enable_lbpw() argument
6193 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) in cik_wait_for_rlc_serdes() argument
6198 mutex_lock(&rdev->grbm_idx_mutex); in cik_wait_for_rlc_serdes()
6199 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { in cik_wait_for_rlc_serdes()
6200 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { in cik_wait_for_rlc_serdes()
6201 cik_select_se_sh(rdev, i, j); in cik_wait_for_rlc_serdes()
6202 for (k = 0; k < rdev->usec_timeout; k++) { in cik_wait_for_rlc_serdes()
6209 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_wait_for_rlc_serdes()
6210 mutex_unlock(&rdev->grbm_idx_mutex); in cik_wait_for_rlc_serdes()
6213 for (k = 0; k < rdev->usec_timeout; k++) { in cik_wait_for_rlc_serdes()
6220 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) in cik_update_rlc() argument
6229 static u32 cik_halt_rlc(struct radeon_device *rdev) in cik_halt_rlc() argument
6241 for (i = 0; i < rdev->usec_timeout; i++) { in cik_halt_rlc()
6247 cik_wait_for_rlc_serdes(rdev); in cik_halt_rlc()
6253 void cik_enter_rlc_safe_mode(struct radeon_device *rdev) in cik_enter_rlc_safe_mode() argument
6261 for (i = 0; i < rdev->usec_timeout; i++) { in cik_enter_rlc_safe_mode()
6267 for (i = 0; i < rdev->usec_timeout; i++) { in cik_enter_rlc_safe_mode()
6274 void cik_exit_rlc_safe_mode(struct radeon_device *rdev) in cik_exit_rlc_safe_mode() argument
6289 static void cik_rlc_stop(struct radeon_device *rdev) in cik_rlc_stop() argument
6293 cik_enable_gui_idle_interrupt(rdev, false); in cik_rlc_stop()
6295 cik_wait_for_rlc_serdes(rdev); in cik_rlc_stop()
6305 static void cik_rlc_start(struct radeon_device *rdev) in cik_rlc_start() argument
6309 cik_enable_gui_idle_interrupt(rdev, true); in cik_rlc_start()
6323 static int cik_rlc_resume(struct radeon_device *rdev) in cik_rlc_resume() argument
6327 if (!rdev->rlc_fw) in cik_rlc_resume()
6330 cik_rlc_stop(rdev); in cik_rlc_resume()
6336 si_rlc_reset(rdev); in cik_rlc_resume()
6338 cik_init_pg(rdev); in cik_rlc_resume()
6340 cik_init_cg(rdev); in cik_rlc_resume()
6345 mutex_lock(&rdev->grbm_idx_mutex); in cik_rlc_resume()
6346 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_rlc_resume()
6350 mutex_unlock(&rdev->grbm_idx_mutex); in cik_rlc_resume()
6355 if (rdev->new_fw) { in cik_rlc_resume()
6357 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; in cik_rlc_resume()
6359 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_rlc_resume()
6371 switch (rdev->family) { in cik_rlc_resume()
6388 fw_data = (const __be32 *)rdev->rlc_fw->data; in cik_rlc_resume()
6396 cik_enable_lbpw(rdev, false); in cik_rlc_resume()
6398 if (rdev->family == CHIP_BONAIRE) in cik_rlc_resume()
6401 cik_rlc_start(rdev); in cik_rlc_resume()
6406 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) in cik_enable_cgcg() argument
6412 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { in cik_enable_cgcg()
6413 cik_enable_gui_idle_interrupt(rdev, true); in cik_enable_cgcg()
6415 tmp = cik_halt_rlc(rdev); in cik_enable_cgcg()
6417 mutex_lock(&rdev->grbm_idx_mutex); in cik_enable_cgcg()
6418 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_enable_cgcg()
6423 mutex_unlock(&rdev->grbm_idx_mutex); in cik_enable_cgcg()
6425 cik_update_rlc(rdev, tmp); in cik_enable_cgcg()
6429 cik_enable_gui_idle_interrupt(rdev, false); in cik_enable_cgcg()
6444 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) in cik_enable_mgcg() argument
6448 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { in cik_enable_mgcg()
6449 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { in cik_enable_mgcg()
6450 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { in cik_enable_mgcg()
6464 tmp = cik_halt_rlc(rdev); in cik_enable_mgcg()
6466 mutex_lock(&rdev->grbm_idx_mutex); in cik_enable_mgcg()
6467 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_enable_mgcg()
6472 mutex_unlock(&rdev->grbm_idx_mutex); in cik_enable_mgcg()
6474 cik_update_rlc(rdev, tmp); in cik_enable_mgcg()
6476 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { in cik_enable_mgcg()
6482 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && in cik_enable_mgcg()
6483 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) in cik_enable_mgcg()
6514 tmp = cik_halt_rlc(rdev); in cik_enable_mgcg()
6516 mutex_lock(&rdev->grbm_idx_mutex); in cik_enable_mgcg()
6517 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_enable_mgcg()
6522 mutex_unlock(&rdev->grbm_idx_mutex); in cik_enable_mgcg()
6524 cik_update_rlc(rdev, tmp); in cik_enable_mgcg()
6541 static void cik_enable_mc_ls(struct radeon_device *rdev, in cik_enable_mc_ls() argument
6549 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) in cik_enable_mc_ls()
6558 static void cik_enable_mc_mgcg(struct radeon_device *rdev, in cik_enable_mc_mgcg() argument
6566 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) in cik_enable_mc_mgcg()
6575 static void cik_enable_sdma_mgcg(struct radeon_device *rdev, in cik_enable_sdma_mgcg() argument
6580 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { in cik_enable_sdma_mgcg()
6596 static void cik_enable_sdma_mgls(struct radeon_device *rdev, in cik_enable_sdma_mgls() argument
6601 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { in cik_enable_sdma_mgls()
6624 static void cik_enable_uvd_mgcg(struct radeon_device *rdev, in cik_enable_uvd_mgcg() argument
6629 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { in cik_enable_uvd_mgcg()
6650 static void cik_enable_bif_mgls(struct radeon_device *rdev, in cik_enable_bif_mgls() argument
6657 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) in cik_enable_bif_mgls()
6668 static void cik_enable_hdp_mgcg(struct radeon_device *rdev, in cik_enable_hdp_mgcg() argument
6675 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) in cik_enable_hdp_mgcg()
6684 static void cik_enable_hdp_ls(struct radeon_device *rdev, in cik_enable_hdp_ls() argument
6691 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) in cik_enable_hdp_ls()
6700 void cik_update_cg(struct radeon_device *rdev, in cik_update_cg() argument
6705 cik_enable_gui_idle_interrupt(rdev, false); in cik_update_cg()
6708 cik_enable_mgcg(rdev, true); in cik_update_cg()
6709 cik_enable_cgcg(rdev, true); in cik_update_cg()
6711 cik_enable_cgcg(rdev, false); in cik_update_cg()
6712 cik_enable_mgcg(rdev, false); in cik_update_cg()
6714 cik_enable_gui_idle_interrupt(rdev, true); in cik_update_cg()
6718 if (!(rdev->flags & RADEON_IS_IGP)) { in cik_update_cg()
6719 cik_enable_mc_mgcg(rdev, enable); in cik_update_cg()
6720 cik_enable_mc_ls(rdev, enable); in cik_update_cg()
6725 cik_enable_sdma_mgcg(rdev, enable); in cik_update_cg()
6726 cik_enable_sdma_mgls(rdev, enable); in cik_update_cg()
6730 cik_enable_bif_mgls(rdev, enable); in cik_update_cg()
6734 if (rdev->has_uvd) in cik_update_cg()
6735 cik_enable_uvd_mgcg(rdev, enable); in cik_update_cg()
6739 cik_enable_hdp_mgcg(rdev, enable); in cik_update_cg()
6740 cik_enable_hdp_ls(rdev, enable); in cik_update_cg()
6744 vce_v2_0_enable_mgcg(rdev, enable); in cik_update_cg()
6748 static void cik_init_cg(struct radeon_device *rdev) in cik_init_cg() argument
6751 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); in cik_init_cg()
6753 if (rdev->has_uvd) in cik_init_cg()
6754 si_init_uvd_internal_cg(rdev); in cik_init_cg()
6756 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | in cik_init_cg()
6763 static void cik_fini_cg(struct radeon_device *rdev) in cik_fini_cg() argument
6765 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | in cik_fini_cg()
6771 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); in cik_fini_cg()
6774 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, in cik_enable_sck_slowdown_on_pu() argument
6780 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) in cik_enable_sck_slowdown_on_pu()
6788 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, in cik_enable_sck_slowdown_on_pd() argument
6794 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) in cik_enable_sck_slowdown_on_pd()
6802 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) in cik_enable_cp_pg() argument
6807 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) in cik_enable_cp_pg()
6815 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) in cik_enable_gds_pg() argument
6820 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) in cik_enable_gds_pg()
6832 void cik_init_cp_pg_table(struct radeon_device *rdev) in cik_init_cp_pg_table() argument
6839 if (rdev->family == CHIP_KAVERI) in cik_init_cp_pg_table()
6842 if (rdev->rlc.cp_table_ptr == NULL) in cik_init_cp_pg_table()
6846 dst_ptr = rdev->rlc.cp_table_ptr; in cik_init_cp_pg_table()
6848 if (rdev->new_fw) { in cik_init_cp_pg_table()
6853 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; in cik_init_cp_pg_table()
6855 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6859 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; in cik_init_cp_pg_table()
6861 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6865 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; in cik_init_cp_pg_table()
6867 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6871 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; in cik_init_cp_pg_table()
6873 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6877 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; in cik_init_cp_pg_table()
6879 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6894 fw_data = (const __be32 *)rdev->ce_fw->data; in cik_init_cp_pg_table()
6897 fw_data = (const __be32 *)rdev->pfp_fw->data; in cik_init_cp_pg_table()
6900 fw_data = (const __be32 *)rdev->me_fw->data; in cik_init_cp_pg_table()
6903 fw_data = (const __be32 *)rdev->mec_fw->data; in cik_init_cp_pg_table()
6916 static void cik_enable_gfx_cgpg(struct radeon_device *rdev, in cik_enable_gfx_cgpg() argument
6921 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { in cik_enable_gfx_cgpg()
6946 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) in cik_get_cu_active_bitmap() argument
6951 mutex_lock(&rdev->grbm_idx_mutex); in cik_get_cu_active_bitmap()
6952 cik_select_se_sh(rdev, se, sh); in cik_get_cu_active_bitmap()
6955 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_get_cu_active_bitmap()
6956 mutex_unlock(&rdev->grbm_idx_mutex); in cik_get_cu_active_bitmap()
6963 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { in cik_get_cu_active_bitmap()
6971 static void cik_init_ao_cu_mask(struct radeon_device *rdev) in cik_init_ao_cu_mask() argument
6977 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { in cik_init_ao_cu_mask()
6978 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { in cik_init_ao_cu_mask()
6982 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { in cik_init_ao_cu_mask()
6983 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { in cik_init_ao_cu_mask()
7004 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, in cik_enable_gfx_static_mgpg() argument
7010 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) in cik_enable_gfx_static_mgpg()
7018 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, in cik_enable_gfx_dynamic_mgpg() argument
7024 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) in cik_enable_gfx_dynamic_mgpg()
7035 static void cik_init_gfx_cgpg(struct radeon_device *rdev) in cik_init_gfx_cgpg() argument
7040 if (rdev->rlc.cs_data) { in cik_init_gfx_cgpg()
7042 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); in cik_init_gfx_cgpg()
7043 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); in cik_init_gfx_cgpg()
7044 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); in cik_init_gfx_cgpg()
7050 if (rdev->rlc.reg_list) { in cik_init_gfx_cgpg()
7052 for (i = 0; i < rdev->rlc.reg_list_size; i++) in cik_init_gfx_cgpg()
7053 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); in cik_init_gfx_cgpg()
7061 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); in cik_init_gfx_cgpg()
7062 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); in cik_init_gfx_cgpg()
7084 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) in cik_update_gfx_pg() argument
7086 cik_enable_gfx_cgpg(rdev, enable); in cik_update_gfx_pg()
7087 cik_enable_gfx_static_mgpg(rdev, enable); in cik_update_gfx_pg()
7088 cik_enable_gfx_dynamic_mgpg(rdev, enable); in cik_update_gfx_pg()
7091 u32 cik_get_csb_size(struct radeon_device *rdev) in cik_get_csb_size() argument
7097 if (rdev->rlc.cs_data == NULL) in cik_get_csb_size()
7105 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { in cik_get_csb_size()
7123 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) in cik_get_csb_buffer() argument
7129 if (rdev->rlc.cs_data == NULL) in cik_get_csb_buffer()
7141 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { in cik_get_csb_buffer()
7157 switch (rdev->family) { in cik_get_csb_buffer()
7188 static void cik_init_pg(struct radeon_device *rdev) in cik_init_pg() argument
7190 if (rdev->pg_flags) { in cik_init_pg()
7191 cik_enable_sck_slowdown_on_pu(rdev, true); in cik_init_pg()
7192 cik_enable_sck_slowdown_on_pd(rdev, true); in cik_init_pg()
7193 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { in cik_init_pg()
7194 cik_init_gfx_cgpg(rdev); in cik_init_pg()
7195 cik_enable_cp_pg(rdev, true); in cik_init_pg()
7196 cik_enable_gds_pg(rdev, true); in cik_init_pg()
7198 cik_init_ao_cu_mask(rdev); in cik_init_pg()
7199 cik_update_gfx_pg(rdev, true); in cik_init_pg()
7203 static void cik_fini_pg(struct radeon_device *rdev) in cik_fini_pg() argument
7205 if (rdev->pg_flags) { in cik_fini_pg()
7206 cik_update_gfx_pg(rdev, false); in cik_fini_pg()
7207 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { in cik_fini_pg()
7208 cik_enable_cp_pg(rdev, false); in cik_fini_pg()
7209 cik_enable_gds_pg(rdev, false); in cik_fini_pg()
7236 static void cik_enable_interrupts(struct radeon_device *rdev) in cik_enable_interrupts() argument
7245 rdev->ih.enabled = true; in cik_enable_interrupts()
7255 static void cik_disable_interrupts(struct radeon_device *rdev) in cik_disable_interrupts() argument
7267 rdev->ih.enabled = false; in cik_disable_interrupts()
7268 rdev->ih.rptr = 0; in cik_disable_interrupts()
7278 static void cik_disable_interrupt_state(struct radeon_device *rdev) in cik_disable_interrupt_state() argument
7307 if (rdev->num_crtc >= 4) { in cik_disable_interrupt_state()
7311 if (rdev->num_crtc >= 6) { in cik_disable_interrupt_state()
7316 if (rdev->num_crtc >= 2) { in cik_disable_interrupt_state()
7320 if (rdev->num_crtc >= 4) { in cik_disable_interrupt_state()
7324 if (rdev->num_crtc >= 6) { in cik_disable_interrupt_state()
7359 static int cik_irq_init(struct radeon_device *rdev) in cik_irq_init() argument
7366 ret = r600_ih_ring_alloc(rdev); in cik_irq_init()
7371 cik_disable_interrupts(rdev); in cik_irq_init()
7374 ret = cik_rlc_resume(rdev); in cik_irq_init()
7376 r600_ih_ring_fini(rdev); in cik_irq_init()
7382 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); in cik_irq_init()
7392 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); in cik_irq_init()
7393 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); in cik_irq_init()
7399 if (rdev->wb.enabled) in cik_irq_init()
7403 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); in cik_irq_init()
7404 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); in cik_irq_init()
7415 if (rdev->msi_enabled) in cik_irq_init()
7420 cik_disable_interrupt_state(rdev); in cik_irq_init()
7422 pci_set_master(rdev->pdev); in cik_irq_init()
7425 cik_enable_interrupts(rdev); in cik_irq_init()
7439 int cik_irq_set(struct radeon_device *rdev) in cik_irq_set() argument
7448 if (!rdev->irq.installed) { in cik_irq_set()
7453 if (!rdev->ih.enabled) { in cik_irq_set()
7454 cik_disable_interrupts(rdev); in cik_irq_set()
7456 cik_disable_interrupt_state(rdev); in cik_irq_set()
7477 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { in cik_irq_set()
7481 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { in cik_irq_set()
7482 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in cik_irq_set()
7497 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { in cik_irq_set()
7498 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in cik_irq_set()
7514 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { in cik_irq_set()
7519 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { in cik_irq_set()
7524 if (rdev->irq.crtc_vblank_int[0] || in cik_irq_set()
7525 atomic_read(&rdev->irq.pflip[0])) { in cik_irq_set()
7529 if (rdev->irq.crtc_vblank_int[1] || in cik_irq_set()
7530 atomic_read(&rdev->irq.pflip[1])) { in cik_irq_set()
7534 if (rdev->irq.crtc_vblank_int[2] || in cik_irq_set()
7535 atomic_read(&rdev->irq.pflip[2])) { in cik_irq_set()
7539 if (rdev->irq.crtc_vblank_int[3] || in cik_irq_set()
7540 atomic_read(&rdev->irq.pflip[3])) { in cik_irq_set()
7544 if (rdev->irq.crtc_vblank_int[4] || in cik_irq_set()
7545 atomic_read(&rdev->irq.pflip[4])) { in cik_irq_set()
7549 if (rdev->irq.crtc_vblank_int[5] || in cik_irq_set()
7550 atomic_read(&rdev->irq.pflip[5])) { in cik_irq_set()
7554 if (rdev->irq.hpd[0]) { in cik_irq_set()
7558 if (rdev->irq.hpd[1]) { in cik_irq_set()
7562 if (rdev->irq.hpd[2]) { in cik_irq_set()
7566 if (rdev->irq.hpd[3]) { in cik_irq_set()
7570 if (rdev->irq.hpd[4]) { in cik_irq_set()
7574 if (rdev->irq.hpd[5]) { in cik_irq_set()
7590 if (rdev->num_crtc >= 4) { in cik_irq_set()
7594 if (rdev->num_crtc >= 6) { in cik_irq_set()
7599 if (rdev->num_crtc >= 2) { in cik_irq_set()
7605 if (rdev->num_crtc >= 4) { in cik_irq_set()
7611 if (rdev->num_crtc >= 6) { in cik_irq_set()
7640 static inline void cik_irq_ack(struct radeon_device *rdev) in cik_irq_ack() argument
7644 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); in cik_irq_ack()
7645 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); in cik_irq_ack()
7646 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); in cik_irq_ack()
7647 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); in cik_irq_ack()
7648 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); in cik_irq_ack()
7649 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); in cik_irq_ack()
7650 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); in cik_irq_ack()
7652 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7654 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7656 if (rdev->num_crtc >= 4) { in cik_irq_ack()
7657 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7659 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7662 if (rdev->num_crtc >= 6) { in cik_irq_ack()
7663 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7665 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7669 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7672 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7675 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) in cik_irq_ack()
7677 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) in cik_irq_ack()
7679 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) in cik_irq_ack()
7681 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) in cik_irq_ack()
7684 if (rdev->num_crtc >= 4) { in cik_irq_ack()
7685 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7688 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7691 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) in cik_irq_ack()
7693 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) in cik_irq_ack()
7695 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) in cik_irq_ack()
7697 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) in cik_irq_ack()
7701 if (rdev->num_crtc >= 6) { in cik_irq_ack()
7702 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7705 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7708 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) in cik_irq_ack()
7710 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) in cik_irq_ack()
7712 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) in cik_irq_ack()
7714 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) in cik_irq_ack()
7718 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { in cik_irq_ack()
7723 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { in cik_irq_ack()
7728 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { in cik_irq_ack()
7733 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { in cik_irq_ack()
7738 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { in cik_irq_ack()
7743 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { in cik_irq_ack()
7748 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) { in cik_irq_ack()
7753 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) { in cik_irq_ack()
7758 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) { in cik_irq_ack()
7763 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) { in cik_irq_ack()
7768 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) { in cik_irq_ack()
7773 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) { in cik_irq_ack()
7787 static void cik_irq_disable(struct radeon_device *rdev) in cik_irq_disable() argument
7789 cik_disable_interrupts(rdev); in cik_irq_disable()
7792 cik_irq_ack(rdev); in cik_irq_disable()
7793 cik_disable_interrupt_state(rdev); in cik_irq_disable()
7804 static void cik_irq_suspend(struct radeon_device *rdev) in cik_irq_suspend() argument
7806 cik_irq_disable(rdev); in cik_irq_suspend()
7807 cik_rlc_stop(rdev); in cik_irq_suspend()
7819 static void cik_irq_fini(struct radeon_device *rdev) in cik_irq_fini() argument
7821 cik_irq_suspend(rdev); in cik_irq_fini()
7822 r600_ih_ring_fini(rdev); in cik_irq_fini()
7836 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) in cik_get_ih_wptr() argument
7840 if (rdev->wb.enabled) in cik_get_ih_wptr()
7841 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); in cik_get_ih_wptr()
7851 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", in cik_get_ih_wptr()
7852 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); in cik_get_ih_wptr()
7853 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; in cik_get_ih_wptr()
7858 return (wptr & rdev->ih.ptr_mask); in cik_get_ih_wptr()
7893 int cik_irq_process(struct radeon_device *rdev) in cik_irq_process() argument
7895 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in cik_irq_process()
7896 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in cik_irq_process()
7908 if (!rdev->ih.enabled || rdev->shutdown) in cik_irq_process()
7911 wptr = cik_get_ih_wptr(rdev); in cik_irq_process()
7915 if (atomic_xchg(&rdev->ih.lock, 1)) in cik_irq_process()
7918 rptr = rdev->ih.rptr; in cik_irq_process()
7925 cik_irq_ack(rdev); in cik_irq_process()
7931 radeon_kfd_interrupt(rdev, in cik_irq_process()
7932 (const void *) &rdev->ih.ring[ring_index]); in cik_irq_process()
7934 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; in cik_irq_process()
7935 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; in cik_irq_process()
7936 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; in cik_irq_process()
7942 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)) in cik_irq_process()
7945 if (rdev->irq.crtc_vblank_int[0]) { in cik_irq_process()
7946 drm_handle_vblank(rdev->ddev, 0); in cik_irq_process()
7947 rdev->pm.vblank_sync = true; in cik_irq_process()
7948 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
7950 if (atomic_read(&rdev->irq.pflip[0])) in cik_irq_process()
7951 radeon_crtc_handle_vblank(rdev, 0); in cik_irq_process()
7952 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; in cik_irq_process()
7957 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)) in cik_irq_process()
7960 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; in cik_irq_process()
7972 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)) in cik_irq_process()
7975 if (rdev->irq.crtc_vblank_int[1]) { in cik_irq_process()
7976 drm_handle_vblank(rdev->ddev, 1); in cik_irq_process()
7977 rdev->pm.vblank_sync = true; in cik_irq_process()
7978 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
7980 if (atomic_read(&rdev->irq.pflip[1])) in cik_irq_process()
7981 radeon_crtc_handle_vblank(rdev, 1); in cik_irq_process()
7982 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; in cik_irq_process()
7987 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)) in cik_irq_process()
7990 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; in cik_irq_process()
8002 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)) in cik_irq_process()
8005 if (rdev->irq.crtc_vblank_int[2]) { in cik_irq_process()
8006 drm_handle_vblank(rdev->ddev, 2); in cik_irq_process()
8007 rdev->pm.vblank_sync = true; in cik_irq_process()
8008 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
8010 if (atomic_read(&rdev->irq.pflip[2])) in cik_irq_process()
8011 radeon_crtc_handle_vblank(rdev, 2); in cik_irq_process()
8012 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; in cik_irq_process()
8017 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)) in cik_irq_process()
8020 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; in cik_irq_process()
8032 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)) in cik_irq_process()
8035 if (rdev->irq.crtc_vblank_int[3]) { in cik_irq_process()
8036 drm_handle_vblank(rdev->ddev, 3); in cik_irq_process()
8037 rdev->pm.vblank_sync = true; in cik_irq_process()
8038 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
8040 if (atomic_read(&rdev->irq.pflip[3])) in cik_irq_process()
8041 radeon_crtc_handle_vblank(rdev, 3); in cik_irq_process()
8042 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; in cik_irq_process()
8047 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)) in cik_irq_process()
8050 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; in cik_irq_process()
8062 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)) in cik_irq_process()
8065 if (rdev->irq.crtc_vblank_int[4]) { in cik_irq_process()
8066 drm_handle_vblank(rdev->ddev, 4); in cik_irq_process()
8067 rdev->pm.vblank_sync = true; in cik_irq_process()
8068 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
8070 if (atomic_read(&rdev->irq.pflip[4])) in cik_irq_process()
8071 radeon_crtc_handle_vblank(rdev, 4); in cik_irq_process()
8072 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; in cik_irq_process()
8077 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)) in cik_irq_process()
8080 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; in cik_irq_process()
8092 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)) in cik_irq_process()
8095 if (rdev->irq.crtc_vblank_int[5]) { in cik_irq_process()
8096 drm_handle_vblank(rdev->ddev, 5); in cik_irq_process()
8097 rdev->pm.vblank_sync = true; in cik_irq_process()
8098 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
8100 if (atomic_read(&rdev->irq.pflip[5])) in cik_irq_process()
8101 radeon_crtc_handle_vblank(rdev, 5); in cik_irq_process()
8102 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; in cik_irq_process()
8107 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)) in cik_irq_process()
8110 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; in cik_irq_process()
8127 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); in cik_irq_process()
8132 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT)) in cik_irq_process()
8135 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; in cik_irq_process()
8141 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT)) in cik_irq_process()
8144 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; in cik_irq_process()
8150 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT)) in cik_irq_process()
8153 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; in cik_irq_process()
8159 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT)) in cik_irq_process()
8162 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; in cik_irq_process()
8168 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT)) in cik_irq_process()
8171 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; in cik_irq_process()
8177 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT)) in cik_irq_process()
8180 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; in cik_irq_process()
8186 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT)) in cik_irq_process()
8189 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT; in cik_irq_process()
8195 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT)) in cik_irq_process()
8198 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT; in cik_irq_process()
8204 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT)) in cik_irq_process()
8207 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT; in cik_irq_process()
8213 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT)) in cik_irq_process()
8216 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT; in cik_irq_process()
8222 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT)) in cik_irq_process()
8225 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT; in cik_irq_process()
8231 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT)) in cik_irq_process()
8234 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT; in cik_irq_process()
8250 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); in cik_irq_process()
8261 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); in cik_irq_process()
8262 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", in cik_irq_process()
8264 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", in cik_irq_process()
8266 cik_vm_decode_fault(rdev, status, addr, mc_client); in cik_irq_process()
8272 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); in cik_irq_process()
8275 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); in cik_irq_process()
8284 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); in cik_irq_process()
8294 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); in cik_irq_process()
8299 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); in cik_irq_process()
8301 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); in cik_irq_process()
8360 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); in cik_irq_process()
8373 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); in cik_irq_process()
8387 rdev->pm.dpm.thermal.high_to_low = false; in cik_irq_process()
8392 rdev->pm.dpm.thermal.high_to_low = true; in cik_irq_process()
8444 rptr &= rdev->ih.ptr_mask; in cik_irq_process()
8448 schedule_work(&rdev->dp_work); in cik_irq_process()
8450 schedule_work(&rdev->hotplug_work); in cik_irq_process()
8452 rdev->needs_reset = true; in cik_irq_process()
8453 wake_up_all(&rdev->fence_queue); in cik_irq_process()
8456 schedule_work(&rdev->pm.dpm.thermal.work); in cik_irq_process()
8457 rdev->ih.rptr = rptr; in cik_irq_process()
8458 atomic_set(&rdev->ih.lock, 0); in cik_irq_process()
8461 wptr = cik_get_ih_wptr(rdev); in cik_irq_process()
8480 static int cik_startup(struct radeon_device *rdev) in cik_startup() argument
8487 cik_pcie_gen3_enable(rdev); in cik_startup()
8489 cik_program_aspm(rdev); in cik_startup()
8492 r = r600_vram_scratch_init(rdev); in cik_startup()
8496 cik_mc_program(rdev); in cik_startup()
8498 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { in cik_startup()
8499 r = ci_mc_load_microcode(rdev); in cik_startup()
8506 r = cik_pcie_gart_enable(rdev); in cik_startup()
8509 cik_gpu_init(rdev); in cik_startup()
8512 if (rdev->flags & RADEON_IS_IGP) { in cik_startup()
8513 if (rdev->family == CHIP_KAVERI) { in cik_startup()
8514 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; in cik_startup()
8515 rdev->rlc.reg_list_size = in cik_startup()
8518 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; in cik_startup()
8519 rdev->rlc.reg_list_size = in cik_startup()
8523 rdev->rlc.cs_data = ci_cs_data; in cik_startup()
8524 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4; in cik_startup()
8525 r = sumo_rlc_init(rdev); in cik_startup()
8532 r = radeon_wb_init(rdev); in cik_startup()
8537 r = cik_mec_init(rdev); in cik_startup()
8543 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); in cik_startup()
8545 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); in cik_startup()
8549 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); in cik_startup()
8551 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); in cik_startup()
8555 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); in cik_startup()
8557 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); in cik_startup()
8561 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); in cik_startup()
8563 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); in cik_startup()
8567 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); in cik_startup()
8569 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); in cik_startup()
8573 r = radeon_uvd_resume(rdev); in cik_startup()
8575 r = uvd_v4_2_resume(rdev); in cik_startup()
8577 r = radeon_fence_driver_start_ring(rdev, in cik_startup()
8580 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); in cik_startup()
8584 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; in cik_startup()
8586 r = radeon_vce_resume(rdev); in cik_startup()
8588 r = vce_v2_0_resume(rdev); in cik_startup()
8590 r = radeon_fence_driver_start_ring(rdev, in cik_startup()
8593 r = radeon_fence_driver_start_ring(rdev, in cik_startup()
8597 dev_err(rdev->dev, "VCE init error (%d).\n", r); in cik_startup()
8598 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; in cik_startup()
8599 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; in cik_startup()
8603 if (!rdev->irq.installed) { in cik_startup()
8604 r = radeon_irq_kms_init(rdev); in cik_startup()
8609 r = cik_irq_init(rdev); in cik_startup()
8612 radeon_irq_kms_fini(rdev); in cik_startup()
8615 cik_irq_set(rdev); in cik_startup()
8617 if (rdev->family == CHIP_HAWAII) { in cik_startup()
8618 if (rdev->new_fw) in cik_startup()
8626 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in cik_startup()
8627 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, in cik_startup()
8634 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in cik_startup()
8635 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, in cik_startup()
8645 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in cik_startup()
8646 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, in cik_startup()
8656 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; in cik_startup()
8657 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, in cik_startup()
8662 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; in cik_startup()
8663 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, in cik_startup()
8668 r = cik_cp_resume(rdev); in cik_startup()
8672 r = cik_sdma_resume(rdev); in cik_startup()
8676 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; in cik_startup()
8678 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, in cik_startup()
8681 r = uvd_v1_0_init(rdev); in cik_startup()
8688 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; in cik_startup()
8690 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, in cik_startup()
8693 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; in cik_startup()
8695 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, in cik_startup()
8699 r = vce_v1_0_init(rdev); in cik_startup()
8703 r = radeon_ib_pool_init(rdev); in cik_startup()
8705 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); in cik_startup()
8709 r = radeon_vm_manager_init(rdev); in cik_startup()
8711 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); in cik_startup()
8715 r = radeon_audio_init(rdev); in cik_startup()
8719 r = radeon_kfd_resume(rdev); in cik_startup()
8735 int cik_resume(struct radeon_device *rdev) in cik_resume() argument
8740 atom_asic_init(rdev->mode_info.atom_context); in cik_resume()
8743 cik_init_golden_registers(rdev); in cik_resume()
8745 if (rdev->pm.pm_method == PM_METHOD_DPM) in cik_resume()
8746 radeon_pm_resume(rdev); in cik_resume()
8748 rdev->accel_working = true; in cik_resume()
8749 r = cik_startup(rdev); in cik_resume()
8752 rdev->accel_working = false; in cik_resume()
8769 int cik_suspend(struct radeon_device *rdev) in cik_suspend() argument
8771 radeon_kfd_suspend(rdev); in cik_suspend()
8772 radeon_pm_suspend(rdev); in cik_suspend()
8773 radeon_audio_fini(rdev); in cik_suspend()
8774 radeon_vm_manager_fini(rdev); in cik_suspend()
8775 cik_cp_enable(rdev, false); in cik_suspend()
8776 cik_sdma_enable(rdev, false); in cik_suspend()
8777 uvd_v1_0_fini(rdev); in cik_suspend()
8778 radeon_uvd_suspend(rdev); in cik_suspend()
8779 radeon_vce_suspend(rdev); in cik_suspend()
8780 cik_fini_pg(rdev); in cik_suspend()
8781 cik_fini_cg(rdev); in cik_suspend()
8782 cik_irq_suspend(rdev); in cik_suspend()
8783 radeon_wb_disable(rdev); in cik_suspend()
8784 cik_pcie_gart_disable(rdev); in cik_suspend()
8804 int cik_init(struct radeon_device *rdev) in cik_init() argument
8810 if (!radeon_get_bios(rdev)) { in cik_init()
8811 if (ASIC_IS_AVIVO(rdev)) in cik_init()
8815 if (!rdev->is_atom_bios) { in cik_init()
8816 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); in cik_init()
8819 r = radeon_atombios_init(rdev); in cik_init()
8824 if (!radeon_card_posted(rdev)) { in cik_init()
8825 if (!rdev->bios) { in cik_init()
8826 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); in cik_init()
8830 atom_asic_init(rdev->mode_info.atom_context); in cik_init()
8833 cik_init_golden_registers(rdev); in cik_init()
8835 cik_scratch_init(rdev); in cik_init()
8837 radeon_surface_init(rdev); in cik_init()
8839 radeon_get_clock_info(rdev->ddev); in cik_init()
8842 r = radeon_fence_driver_init(rdev); in cik_init()
8847 r = cik_mc_init(rdev); in cik_init()
8851 r = radeon_bo_init(rdev); in cik_init()
8855 if (rdev->flags & RADEON_IS_IGP) { in cik_init()
8856 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || in cik_init()
8857 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { in cik_init()
8858 r = cik_init_microcode(rdev); in cik_init()
8865 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || in cik_init()
8866 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || in cik_init()
8867 !rdev->mc_fw) { in cik_init()
8868 r = cik_init_microcode(rdev); in cik_init()
8877 radeon_pm_init(rdev); in cik_init()
8879 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in cik_init()
8881 r600_ring_init(rdev, ring, 1024 * 1024); in cik_init()
8883 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in cik_init()
8885 r600_ring_init(rdev, ring, 1024 * 1024); in cik_init()
8886 r = radeon_doorbell_get(rdev, &ring->doorbell_index); in cik_init()
8890 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in cik_init()
8892 r600_ring_init(rdev, ring, 1024 * 1024); in cik_init()
8893 r = radeon_doorbell_get(rdev, &ring->doorbell_index); in cik_init()
8897 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; in cik_init()
8899 r600_ring_init(rdev, ring, 256 * 1024); in cik_init()
8901 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; in cik_init()
8903 r600_ring_init(rdev, ring, 256 * 1024); in cik_init()
8905 r = radeon_uvd_init(rdev); in cik_init()
8907 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; in cik_init()
8909 r600_ring_init(rdev, ring, 4096); in cik_init()
8912 r = radeon_vce_init(rdev); in cik_init()
8914 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; in cik_init()
8916 r600_ring_init(rdev, ring, 4096); in cik_init()
8918 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; in cik_init()
8920 r600_ring_init(rdev, ring, 4096); in cik_init()
8923 rdev->ih.ring_obj = NULL; in cik_init()
8924 r600_ih_ring_init(rdev, 64 * 1024); in cik_init()
8926 r = r600_pcie_gart_init(rdev); in cik_init()
8930 rdev->accel_working = true; in cik_init()
8931 r = cik_startup(rdev); in cik_init()
8933 dev_err(rdev->dev, "disabling GPU acceleration\n"); in cik_init()
8934 cik_cp_fini(rdev); in cik_init()
8935 cik_sdma_fini(rdev); in cik_init()
8936 cik_irq_fini(rdev); in cik_init()
8937 sumo_rlc_fini(rdev); in cik_init()
8938 cik_mec_fini(rdev); in cik_init()
8939 radeon_wb_fini(rdev); in cik_init()
8940 radeon_ib_pool_fini(rdev); in cik_init()
8941 radeon_vm_manager_fini(rdev); in cik_init()
8942 radeon_irq_kms_fini(rdev); in cik_init()
8943 cik_pcie_gart_fini(rdev); in cik_init()
8944 rdev->accel_working = false; in cik_init()
8951 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { in cik_init()
8968 void cik_fini(struct radeon_device *rdev) in cik_fini() argument
8970 radeon_pm_fini(rdev); in cik_fini()
8971 cik_cp_fini(rdev); in cik_fini()
8972 cik_sdma_fini(rdev); in cik_fini()
8973 cik_fini_pg(rdev); in cik_fini()
8974 cik_fini_cg(rdev); in cik_fini()
8975 cik_irq_fini(rdev); in cik_fini()
8976 sumo_rlc_fini(rdev); in cik_fini()
8977 cik_mec_fini(rdev); in cik_fini()
8978 radeon_wb_fini(rdev); in cik_fini()
8979 radeon_vm_manager_fini(rdev); in cik_fini()
8980 radeon_ib_pool_fini(rdev); in cik_fini()
8981 radeon_irq_kms_fini(rdev); in cik_fini()
8982 uvd_v1_0_fini(rdev); in cik_fini()
8983 radeon_uvd_fini(rdev); in cik_fini()
8984 radeon_vce_fini(rdev); in cik_fini()
8985 cik_pcie_gart_fini(rdev); in cik_fini()
8986 r600_vram_scratch_fini(rdev); in cik_fini()
8987 radeon_gem_fini(rdev); in cik_fini()
8988 radeon_fence_driver_fini(rdev); in cik_fini()
8989 radeon_bo_fini(rdev); in cik_fini()
8990 radeon_atombios_fini(rdev); in cik_fini()
8991 kfree(rdev->bios); in cik_fini()
8992 rdev->bios = NULL; in cik_fini()
8998 struct radeon_device *rdev = dev->dev_private; in dce8_program_fmt() local
9072 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, in dce8_line_buffer_adjust() argument
9095 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; in dce8_line_buffer_adjust()
9099 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; in dce8_line_buffer_adjust()
9111 for (i = 0; i < rdev->usec_timeout; i++) { in dce8_line_buffer_adjust()
9143 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) in cik_get_number_of_dram_channels() argument
9514 static void dce8_program_watermarks(struct radeon_device *rdev, in dce8_program_watermarks() argument
9530 if ((rdev->pm.pm_method == PM_METHOD_DPM) && in dce8_program_watermarks()
9531 rdev->pm.dpm_enabled) { in dce8_program_watermarks()
9533 radeon_dpm_get_mclk(rdev, false) * 10; in dce8_program_watermarks()
9535 radeon_dpm_get_sclk(rdev, false) * 10; in dce8_program_watermarks()
9537 wm_high.yclk = rdev->pm.current_mclk * 10; in dce8_program_watermarks()
9538 wm_high.sclk = rdev->pm.current_sclk * 10; in dce8_program_watermarks()
9554 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); in dce8_program_watermarks()
9565 (rdev->disp_priority == 2)) { in dce8_program_watermarks()
9570 if ((rdev->pm.pm_method == PM_METHOD_DPM) && in dce8_program_watermarks()
9571 rdev->pm.dpm_enabled) { in dce8_program_watermarks()
9573 radeon_dpm_get_mclk(rdev, true) * 10; in dce8_program_watermarks()
9575 radeon_dpm_get_sclk(rdev, true) * 10; in dce8_program_watermarks()
9577 wm_low.yclk = rdev->pm.current_mclk * 10; in dce8_program_watermarks()
9578 wm_low.sclk = rdev->pm.current_sclk * 10; in dce8_program_watermarks()
9594 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); in dce8_program_watermarks()
9605 (rdev->disp_priority == 2)) { in dce8_program_watermarks()
9644 void dce8_bandwidth_update(struct radeon_device *rdev) in dce8_bandwidth_update() argument
9650 if (!rdev->mode_info.mode_config_initialized) in dce8_bandwidth_update()
9653 radeon_update_display_priority(rdev); in dce8_bandwidth_update()
9655 for (i = 0; i < rdev->num_crtc; i++) { in dce8_bandwidth_update()
9656 if (rdev->mode_info.crtcs[i]->base.enabled) in dce8_bandwidth_update()
9659 for (i = 0; i < rdev->num_crtc; i++) { in dce8_bandwidth_update()
9660 mode = &rdev->mode_info.crtcs[i]->base.mode; in dce8_bandwidth_update()
9661 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); in dce8_bandwidth_update()
9662 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); in dce8_bandwidth_update()
9674 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) in cik_get_gpu_clock_counter() argument
9678 mutex_lock(&rdev->gpu_clock_mutex); in cik_get_gpu_clock_counter()
9682 mutex_unlock(&rdev->gpu_clock_mutex); in cik_get_gpu_clock_counter()
9686 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, in cik_set_uvd_clock() argument
9693 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, in cik_set_uvd_clock()
9714 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) in cik_set_uvd_clocks() argument
9718 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); in cik_set_uvd_clocks()
9722 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); in cik_set_uvd_clocks()
9726 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) in cik_set_vce_clocks() argument
9732 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, in cik_set_vce_clocks()
9761 static void cik_pcie_gen3_enable(struct radeon_device *rdev) in cik_pcie_gen3_enable() argument
9763 struct pci_dev *root = rdev->pdev->bus->self; in cik_pcie_gen3_enable()
9769 if (pci_is_root_bus(rdev->pdev->bus)) in cik_pcie_gen3_enable()
9775 if (rdev->flags & RADEON_IS_IGP) in cik_pcie_gen3_enable()
9778 if (!(rdev->flags & RADEON_IS_PCIE)) in cik_pcie_gen3_enable()
9781 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); in cik_pcie_gen3_enable()
9809 gpu_pos = pci_pcie_cap(rdev->pdev); in cik_pcie_gen3_enable()
9821 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); in cik_pcie_gen3_enable()
9827 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); in cik_pcie_gen3_enable()
9845 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); in cik_pcie_gen3_enable()
9850 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); in cik_pcie_gen3_enable()
9853 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); in cik_pcie_gen3_enable()
9871 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); in cik_pcie_gen3_enable()
9874 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); in cik_pcie_gen3_enable()
9882 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); in cik_pcie_gen3_enable()
9885 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); in cik_pcie_gen3_enable()
9899 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); in cik_pcie_gen3_enable()
9907 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); in cik_pcie_gen3_enable()
9913 for (i = 0; i < rdev->usec_timeout; i++) { in cik_pcie_gen3_enable()
9921 static void cik_program_aspm(struct radeon_device *rdev) in cik_program_aspm() argument
9931 if (rdev->flags & RADEON_IS_IGP) in cik_program_aspm()
9934 if (!(rdev->flags & RADEON_IS_PCIE)) in cik_program_aspm()
9999 !pci_is_root_bus(rdev->pdev->bus)) { in cik_program_aspm()
10000 struct pci_dev *root = rdev->pdev->bus->self; in cik_program_aspm()