41 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)  in eg_cg_rreg()  argument
46 	spin_lock_irqsave(&rdev->cg_idx_lock, flags);  in eg_cg_rreg()
49 	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);  in eg_cg_rreg()
53 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in eg_cg_wreg()  argument
57 	spin_lock_irqsave(&rdev->cg_idx_lock, flags);  in eg_cg_wreg()
60 	spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);  in eg_cg_wreg()
63 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)  in eg_pif_phy0_rreg()  argument
68 	spin_lock_irqsave(&rdev->pif_idx_lock, flags);  in eg_pif_phy0_rreg()
71 	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);  in eg_pif_phy0_rreg()
75 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in eg_pif_phy0_wreg()  argument
79 	spin_lock_irqsave(&rdev->pif_idx_lock, flags);  in eg_pif_phy0_wreg()
82 	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);  in eg_pif_phy0_wreg()
85 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)  in eg_pif_phy1_rreg()  argument
90 	spin_lock_irqsave(&rdev->pif_idx_lock, flags);  in eg_pif_phy1_rreg()
93 	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);  in eg_pif_phy1_rreg()
97 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)  in eg_pif_phy1_wreg()  argument
101 	spin_lock_irqsave(&rdev->pif_idx_lock, flags);  in eg_pif_phy1_wreg()
104 	spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);  in eg_pif_phy1_wreg()
204 static void evergreen_gpu_init(struct radeon_device *rdev);
205 void evergreen_fini(struct radeon_device *rdev);
206 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
207 void evergreen_program_aspm(struct radeon_device *rdev);
208 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
210 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
212 void cik_init_cp_pg_table(struct radeon_device *rdev);
214 extern u32 si_get_csb_size(struct radeon_device *rdev);
215 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
216 extern u32 cik_get_csb_size(struct radeon_device *rdev);
217 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
218 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
992 static void evergreen_init_golden_registers(struct radeon_device *rdev)  in evergreen_init_golden_registers()  argument
994 	switch (rdev->family) {  in evergreen_init_golden_registers()
997 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1000 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1003 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1008 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1011 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1014 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1019 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1022 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1025 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1030 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1033 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1036 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1041 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1046 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1051 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1054 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1059 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1064 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1069 		radeon_program_register_sequence(rdev,  in evergreen_init_golden_registers()
1088 int evergreen_get_allowed_info_register(struct radeon_device *rdev,  in evergreen_get_allowed_info_register()  argument
1137 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,  in sumo_set_uvd_clock()  argument
1143         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,  in sumo_set_uvd_clock()
1161 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)  in sumo_set_uvd_clocks()  argument
1166 	r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);  in sumo_set_uvd_clocks()
1172 	r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);  in sumo_set_uvd_clocks()
1184 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)  in evergreen_set_uvd_clocks()  argument
1204 	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,  in evergreen_set_uvd_clocks()
1222 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);  in evergreen_set_uvd_clocks()
1259 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);  in evergreen_set_uvd_clocks()
1273 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)  in evergreen_fix_pci_max_read_req_size()  argument
1278 	readrq = pcie_get_readrq(rdev->pdev);  in evergreen_fix_pci_max_read_req_size()
1284 		pcie_set_readrq(rdev->pdev, 512);  in evergreen_fix_pci_max_read_req_size()
1290 	struct radeon_device *rdev = dev->dev_private;  in dce4_program_fmt()  local
1343 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)  in dce4_is_in_vblank()  argument
1351 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)  in dce4_is_counter_moving()  argument
1372 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)  in dce4_wait_for_vblank()  argument
1376 	if (crtc >= rdev->num_crtc)  in dce4_wait_for_vblank()
1385 	while (dce4_is_in_vblank(rdev, crtc)) {  in dce4_wait_for_vblank()
1387 			if (!dce4_is_counter_moving(rdev, crtc))  in dce4_wait_for_vblank()
1392 	while (!dce4_is_in_vblank(rdev, crtc)) {  in dce4_wait_for_vblank()
1394 			if (!dce4_is_counter_moving(rdev, crtc))  in dce4_wait_for_vblank()
1410 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)  in evergreen_page_flip()  argument
1412 	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];  in evergreen_page_flip()
1431 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)  in evergreen_page_flip_pending()  argument
1433 	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];  in evergreen_page_flip_pending()
1441 int evergreen_get_temp(struct radeon_device *rdev)  in evergreen_get_temp()  argument
1446 	if (rdev->family == CHIP_JUNIPER) {  in evergreen_get_temp()
1479 int sumo_get_temp(struct radeon_device *rdev)  in sumo_get_temp()  argument
1496 void sumo_pm_init_profile(struct radeon_device *rdev)  in sumo_pm_init_profile()  argument
1501 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in sumo_pm_init_profile()
1502 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in sumo_pm_init_profile()
1503 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1504 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1507 	if (rdev->flags & RADEON_IS_MOBILITY)  in sumo_pm_init_profile()
1508 		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);  in sumo_pm_init_profile()
1510 		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);  in sumo_pm_init_profile()
1512 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1513 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1514 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1515 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1517 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1518 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1519 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1520 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1522 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1523 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1524 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1525 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1527 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1528 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1529 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1530 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;  in sumo_pm_init_profile()
1533 	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);  in sumo_pm_init_profile()
1534 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1535 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1536 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1537 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =  in sumo_pm_init_profile()
1538 		rdev->pm.power_state[idx].num_clock_modes - 1;  in sumo_pm_init_profile()
1540 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;  in sumo_pm_init_profile()
1541 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;  in sumo_pm_init_profile()
1542 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in sumo_pm_init_profile()
1543 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =  in sumo_pm_init_profile()
1544 		rdev->pm.power_state[idx].num_clock_modes - 1;  in sumo_pm_init_profile()
1556 void btc_pm_init_profile(struct radeon_device *rdev)  in btc_pm_init_profile()  argument
1561 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;  in btc_pm_init_profile()
1562 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;  in btc_pm_init_profile()
1563 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1564 	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;  in btc_pm_init_profile()
1569 	if (rdev->flags & RADEON_IS_MOBILITY)  in btc_pm_init_profile()
1570 		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);  in btc_pm_init_profile()
1572 		idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);  in btc_pm_init_profile()
1574 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1575 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1576 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1577 	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;  in btc_pm_init_profile()
1579 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1580 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1581 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1582 	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;  in btc_pm_init_profile()
1584 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1585 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1586 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1587 	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;  in btc_pm_init_profile()
1589 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1590 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1591 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1592 	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;  in btc_pm_init_profile()
1594 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1595 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1596 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1597 	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;  in btc_pm_init_profile()
1599 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;  in btc_pm_init_profile()
1600 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;  in btc_pm_init_profile()
1601 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;  in btc_pm_init_profile()
1602 	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;  in btc_pm_init_profile()
1613 void evergreen_pm_misc(struct radeon_device *rdev)  in evergreen_pm_misc()  argument
1615 	int req_ps_idx = rdev->pm.requested_power_state_index;  in evergreen_pm_misc()
1616 	int req_cm_idx = rdev->pm.requested_clock_mode_index;  in evergreen_pm_misc()
1617 	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];  in evergreen_pm_misc()
1624 		if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {  in evergreen_pm_misc()
1625 			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);  in evergreen_pm_misc()
1626 			rdev->pm.current_vddc = voltage->voltage;  in evergreen_pm_misc()
1634 		if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&  in evergreen_pm_misc()
1635 		    (rdev->family >= CHIP_BARTS) &&  in evergreen_pm_misc()
1636 		    rdev->pm.active_crtc_count &&  in evergreen_pm_misc()
1637 		    ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||  in evergreen_pm_misc()
1638 		     (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))  in evergreen_pm_misc()
1639 			voltage = &rdev->pm.power_state[req_ps_idx].  in evergreen_pm_misc()
1640 				clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;  in evergreen_pm_misc()
1645 		if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {  in evergreen_pm_misc()
1646 			radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);  in evergreen_pm_misc()
1647 			rdev->pm.current_vddci = voltage->vddci;  in evergreen_pm_misc()
1660 void evergreen_pm_prepare(struct radeon_device *rdev)  in evergreen_pm_prepare()  argument
1662 	struct drm_device *ddev = rdev->ddev;  in evergreen_pm_prepare()
1685 void evergreen_pm_finish(struct radeon_device *rdev)  in evergreen_pm_finish()  argument
1687 	struct drm_device *ddev = rdev->ddev;  in evergreen_pm_finish()
1712 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)  in evergreen_hpd_sense()  argument
1756 void evergreen_hpd_set_polarity(struct radeon_device *rdev,  in evergreen_hpd_set_polarity()  argument
1760 	bool connected = evergreen_hpd_sense(rdev, hpd);  in evergreen_hpd_set_polarity()
1824 void evergreen_hpd_init(struct radeon_device *rdev)  in evergreen_hpd_init()  argument
1826 	struct drm_device *dev = rdev->ddev;  in evergreen_hpd_init()
1866 		radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);  in evergreen_hpd_init()
1869 	radeon_irq_kms_enable_hpd(rdev, enabled);  in evergreen_hpd_init()
1880 void evergreen_hpd_fini(struct radeon_device *rdev)  in evergreen_hpd_fini()  argument
1882 	struct drm_device *dev = rdev->ddev;  in evergreen_hpd_fini()
1912 	radeon_irq_kms_disable_hpd(rdev, disabled);  in evergreen_hpd_fini()
1917 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,  in evergreen_line_buffer_adjust()  argument
1963 	if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {  in evergreen_line_buffer_adjust()
1966 		for (i = 0; i < rdev->usec_timeout; i++) {  in evergreen_line_buffer_adjust()
1979 			if (ASIC_IS_DCE5(rdev))  in evergreen_line_buffer_adjust()
1985 			if (ASIC_IS_DCE5(rdev))  in evergreen_line_buffer_adjust()
1991 			if (ASIC_IS_DCE5(rdev))  in evergreen_line_buffer_adjust()
1997 			if (ASIC_IS_DCE5(rdev))  in evergreen_line_buffer_adjust()
2008 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)  in evergreen_get_number_of_dram_channels()  argument
2252 static void evergreen_program_watermarks(struct radeon_device *rdev,  in evergreen_program_watermarks()  argument
2274 		dram_channels = evergreen_get_number_of_dram_channels(rdev);  in evergreen_program_watermarks()
2277 		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {  in evergreen_program_watermarks()
2279 				radeon_dpm_get_mclk(rdev, false) * 10;  in evergreen_program_watermarks()
2281 				radeon_dpm_get_sclk(rdev, false) * 10;  in evergreen_program_watermarks()
2283 			wm_high.yclk = rdev->pm.current_mclk * 10;  in evergreen_program_watermarks()
2284 			wm_high.sclk = rdev->pm.current_sclk * 10;  in evergreen_program_watermarks()
2304 		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {  in evergreen_program_watermarks()
2306 				radeon_dpm_get_mclk(rdev, true) * 10;  in evergreen_program_watermarks()
2308 				radeon_dpm_get_sclk(rdev, true) * 10;  in evergreen_program_watermarks()
2310 			wm_low.yclk = rdev->pm.current_mclk * 10;  in evergreen_program_watermarks()
2311 			wm_low.sclk = rdev->pm.current_sclk * 10;  in evergreen_program_watermarks()
2340 		    (rdev->disp_priority == 2)) {  in evergreen_program_watermarks()
2347 		    (rdev->disp_priority == 2)) {  in evergreen_program_watermarks()
2418 void evergreen_bandwidth_update(struct radeon_device *rdev)  in evergreen_bandwidth_update()  argument
2425 	if (!rdev->mode_info.mode_config_initialized)  in evergreen_bandwidth_update()
2428 	radeon_update_display_priority(rdev);  in evergreen_bandwidth_update()
2430 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_bandwidth_update()
2431 		if (rdev->mode_info.crtcs[i]->base.enabled)  in evergreen_bandwidth_update()
2434 	for (i = 0; i < rdev->num_crtc; i += 2) {  in evergreen_bandwidth_update()
2435 		mode0 = &rdev->mode_info.crtcs[i]->base.mode;  in evergreen_bandwidth_update()
2436 		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;  in evergreen_bandwidth_update()
2437 		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);  in evergreen_bandwidth_update()
2438 		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);  in evergreen_bandwidth_update()
2439 		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);  in evergreen_bandwidth_update()
2440 		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);  in evergreen_bandwidth_update()
2453 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)  in evergreen_mc_wait_for_idle()  argument
2458 	for (i = 0; i < rdev->usec_timeout; i++) {  in evergreen_mc_wait_for_idle()
2471 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)  in evergreen_pcie_gart_tlb_flush()  argument
2479 	for (i = 0; i < rdev->usec_timeout; i++) {  in evergreen_pcie_gart_tlb_flush()
2494 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)  in evergreen_pcie_gart_enable()  argument
2499 	if (rdev->gart.robj == NULL) {  in evergreen_pcie_gart_enable()
2500 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");  in evergreen_pcie_gart_enable()
2503 	r = radeon_gart_table_vram_pin(rdev);  in evergreen_pcie_gart_enable()
2517 	if (rdev->flags & RADEON_IS_IGP) {  in evergreen_pcie_gart_enable()
2525 		if ((rdev->family == CHIP_JUNIPER) ||  in evergreen_pcie_gart_enable()
2526 		    (rdev->family == CHIP_CYPRESS) ||  in evergreen_pcie_gart_enable()
2527 		    (rdev->family == CHIP_HEMLOCK) ||  in evergreen_pcie_gart_enable()
2528 		    (rdev->family == CHIP_BARTS))  in evergreen_pcie_gart_enable()
2535 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);  in evergreen_pcie_gart_enable()
2536 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);  in evergreen_pcie_gart_enable()
2537 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);  in evergreen_pcie_gart_enable()
2541 			(u32)(rdev->dummy_page.addr >> 12));  in evergreen_pcie_gart_enable()
2544 	evergreen_pcie_gart_tlb_flush(rdev);  in evergreen_pcie_gart_enable()
2546 		 (unsigned)(rdev->mc.gtt_size >> 20),  in evergreen_pcie_gart_enable()
2547 		 (unsigned long long)rdev->gart.table_addr);  in evergreen_pcie_gart_enable()
2548 	rdev->gart.ready = true;  in evergreen_pcie_gart_enable()
2552 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)  in evergreen_pcie_gart_disable()  argument
2574 	radeon_gart_table_vram_unpin(rdev);  in evergreen_pcie_gart_disable()
2577 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)  in evergreen_pcie_gart_fini()  argument
2579 	evergreen_pcie_gart_disable(rdev);  in evergreen_pcie_gart_fini()
2580 	radeon_gart_table_vram_free(rdev);  in evergreen_pcie_gart_fini()
2581 	radeon_gart_fini(rdev);  in evergreen_pcie_gart_fini()
2585 static void evergreen_agp_enable(struct radeon_device *rdev)  in evergreen_agp_enable()  argument
2652 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,  in evergreen_is_dp_sst_stream_enabled()  argument
2712 static void evergreen_blank_dp_output(struct radeon_device *rdev,  in evergreen_blank_dp_output()  argument
2752 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)  in evergreen_mc_stop()  argument
2758 	if (!ASIC_IS_NODCE(rdev)) {  in evergreen_mc_stop()
2766 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_stop()
2770 			if (ASIC_IS_DCE6(rdev)) {  in evergreen_mc_stop()
2773 					radeon_wait_for_vblank(rdev, i);  in evergreen_mc_stop()
2782 					radeon_wait_for_vblank(rdev, i);  in evergreen_mc_stop()
2790 			frame_count = radeon_get_vblank_counter(rdev, i);  in evergreen_mc_stop()
2791 			for (j = 0; j < rdev->usec_timeout; j++) {  in evergreen_mc_stop()
2792 				if (radeon_get_vblank_counter(rdev, i) != frame_count)  in evergreen_mc_stop()
2803 			if (ASIC_IS_DCE5(rdev) &&  in evergreen_mc_stop()
2804 			    evergreen_is_dp_sst_stream_enabled(rdev, i ,&dig_fe))  in evergreen_mc_stop()
2805 				evergreen_blank_dp_output(rdev, dig_fe);  in evergreen_mc_stop()
2820 	radeon_mc_wait_for_idle(rdev);  in evergreen_mc_stop()
2834 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_stop()
2850 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)  in evergreen_mc_resume()  argument
2856 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_resume()
2858 		       upper_32_bits(rdev->mc.vram_start));  in evergreen_mc_resume()
2860 		       upper_32_bits(rdev->mc.vram_start));  in evergreen_mc_resume()
2862 		       (u32)rdev->mc.vram_start);  in evergreen_mc_resume()
2864 		       (u32)rdev->mc.vram_start);  in evergreen_mc_resume()
2867 	if (!ASIC_IS_NODCE(rdev)) {  in evergreen_mc_resume()
2868 		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));  in evergreen_mc_resume()
2869 		WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);  in evergreen_mc_resume()
2873 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_resume()
2891 			for (j = 0; j < rdev->usec_timeout; j++) {  in evergreen_mc_resume()
2907 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_mc_resume()
2909 			if (ASIC_IS_DCE6(rdev)) {  in evergreen_mc_resume()
2923 			frame_count = radeon_get_vblank_counter(rdev, i);  in evergreen_mc_resume()
2924 			for (j = 0; j < rdev->usec_timeout; j++) {  in evergreen_mc_resume()
2925 				if (radeon_get_vblank_counter(rdev, i) != frame_count)  in evergreen_mc_resume()
2931 	if (!ASIC_IS_NODCE(rdev)) {  in evergreen_mc_resume()
2939 void evergreen_mc_program(struct radeon_device *rdev)  in evergreen_mc_program()  argument
2955 	evergreen_mc_stop(rdev, &save);  in evergreen_mc_program()
2956 	if (evergreen_mc_wait_for_idle(rdev)) {  in evergreen_mc_program()
2957 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in evergreen_mc_program()
2962 	if (rdev->flags & RADEON_IS_AGP) {  in evergreen_mc_program()
2963 		if (rdev->mc.vram_start < rdev->mc.gtt_start) {  in evergreen_mc_program()
2966 				rdev->mc.vram_start >> 12);  in evergreen_mc_program()
2968 				rdev->mc.gtt_end >> 12);  in evergreen_mc_program()
2972 				rdev->mc.gtt_start >> 12);  in evergreen_mc_program()
2974 				rdev->mc.vram_end >> 12);  in evergreen_mc_program()
2978 			rdev->mc.vram_start >> 12);  in evergreen_mc_program()
2980 			rdev->mc.vram_end >> 12);  in evergreen_mc_program()
2982 	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);  in evergreen_mc_program()
2984 	if ((rdev->family == CHIP_PALM) ||  in evergreen_mc_program()
2985 	    (rdev->family == CHIP_SUMO) ||  in evergreen_mc_program()
2986 	    (rdev->family == CHIP_SUMO2)) {  in evergreen_mc_program()
2988 		tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;  in evergreen_mc_program()
2989 		tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;  in evergreen_mc_program()
2992 	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;  in evergreen_mc_program()
2993 	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);  in evergreen_mc_program()
2995 	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));  in evergreen_mc_program()
2998 	if (rdev->flags & RADEON_IS_AGP) {  in evergreen_mc_program()
2999 		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);  in evergreen_mc_program()
3000 		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);  in evergreen_mc_program()
3001 		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);  in evergreen_mc_program()
3007 	if (evergreen_mc_wait_for_idle(rdev)) {  in evergreen_mc_program()
3008 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in evergreen_mc_program()
3010 	evergreen_mc_resume(rdev, &save);  in evergreen_mc_program()
3013 	rv515_vga_render_disable(rdev);  in evergreen_mc_program()
3019 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)  in evergreen_ring_ib_execute()  argument
3021 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in evergreen_ring_ib_execute()
3034 	} else if (rdev->wb.enabled) {  in evergreen_ring_ib_execute()
3054 static int evergreen_cp_load_microcode(struct radeon_device *rdev)  in evergreen_cp_load_microcode()  argument
3059 	if (!rdev->me_fw || !rdev->pfp_fw)  in evergreen_cp_load_microcode()
3062 	r700_cp_stop(rdev);  in evergreen_cp_load_microcode()
3069 	fw_data = (const __be32 *)rdev->pfp_fw->data;  in evergreen_cp_load_microcode()
3075 	fw_data = (const __be32 *)rdev->me_fw->data;  in evergreen_cp_load_microcode()
3086 static int evergreen_cp_start(struct radeon_device *rdev)  in evergreen_cp_start()  argument
3088 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in evergreen_cp_start()
3092 	r = radeon_ring_lock(rdev, ring, 7);  in evergreen_cp_start()
3100 	radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);  in evergreen_cp_start()
3104 	radeon_ring_unlock_commit(rdev, ring, false);  in evergreen_cp_start()
3109 	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);  in evergreen_cp_start()
3147 	radeon_ring_unlock_commit(rdev, ring, false);  in evergreen_cp_start()
3152 static int evergreen_cp_resume(struct radeon_device *rdev)  in evergreen_cp_resume()  argument
3154 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in evergreen_cp_resume()
3192 	       ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));  in evergreen_cp_resume()
3193 	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);  in evergreen_cp_resume()
3194 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);  in evergreen_cp_resume()
3196 	if (rdev->wb.enabled)  in evergreen_cp_resume()
3209 	evergreen_cp_start(rdev);  in evergreen_cp_resume()
3211 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);  in evergreen_cp_resume()
3222 static void evergreen_gpu_init(struct radeon_device *rdev)  in evergreen_gpu_init()  argument
3243 	switch (rdev->family) {  in evergreen_gpu_init()
3246 		rdev->config.evergreen.num_ses = 2;  in evergreen_gpu_init()
3247 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3248 		rdev->config.evergreen.max_tile_pipes = 8;  in evergreen_gpu_init()
3249 		rdev->config.evergreen.max_simds = 10;  in evergreen_gpu_init()
3250 		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3251 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3252 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3253 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3254 		rdev->config.evergreen.max_stack_entries = 512;  in evergreen_gpu_init()
3255 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3256 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3257 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3258 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3259 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3260 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3262 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3263 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3264 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3268 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3269 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3270 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3271 		rdev->config.evergreen.max_simds = 10;  in evergreen_gpu_init()
3272 		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3273 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3274 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3275 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3276 		rdev->config.evergreen.max_stack_entries = 512;  in evergreen_gpu_init()
3277 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3278 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3279 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3280 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3281 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3282 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3284 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3285 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3286 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3290 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3291 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3292 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3293 		rdev->config.evergreen.max_simds = 5;  in evergreen_gpu_init()
3294 		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3295 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3296 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3297 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3298 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3299 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3300 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3301 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3302 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3303 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3304 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3306 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3307 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3308 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3313 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3314 		rdev->config.evergreen.max_pipes = 2;  in evergreen_gpu_init()
3315 		rdev->config.evergreen.max_tile_pipes = 2;  in evergreen_gpu_init()
3316 		rdev->config.evergreen.max_simds = 2;  in evergreen_gpu_init()
3317 		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3318 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3319 		rdev->config.evergreen.max_threads = 192;  in evergreen_gpu_init()
3320 		rdev->config.evergreen.max_gs_threads = 16;  in evergreen_gpu_init()
3321 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3322 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3323 		rdev->config.evergreen.sx_max_export_size = 128;  in evergreen_gpu_init()
3324 		rdev->config.evergreen.sx_max_export_pos_size = 32;  in evergreen_gpu_init()
3325 		rdev->config.evergreen.sx_max_export_smx_size = 96;  in evergreen_gpu_init()
3326 		rdev->config.evergreen.max_hw_contexts = 4;  in evergreen_gpu_init()
3327 		rdev->config.evergreen.sq_num_cf_insts = 1;  in evergreen_gpu_init()
3329 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3330 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3331 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3335 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3336 		rdev->config.evergreen.max_pipes = 2;  in evergreen_gpu_init()
3337 		rdev->config.evergreen.max_tile_pipes = 2;  in evergreen_gpu_init()
3338 		rdev->config.evergreen.max_simds = 2;  in evergreen_gpu_init()
3339 		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3340 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3341 		rdev->config.evergreen.max_threads = 192;  in evergreen_gpu_init()
3342 		rdev->config.evergreen.max_gs_threads = 16;  in evergreen_gpu_init()
3343 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3344 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3345 		rdev->config.evergreen.sx_max_export_size = 128;  in evergreen_gpu_init()
3346 		rdev->config.evergreen.sx_max_export_pos_size = 32;  in evergreen_gpu_init()
3347 		rdev->config.evergreen.sx_max_export_smx_size = 96;  in evergreen_gpu_init()
3348 		rdev->config.evergreen.max_hw_contexts = 4;  in evergreen_gpu_init()
3349 		rdev->config.evergreen.sq_num_cf_insts = 1;  in evergreen_gpu_init()
3351 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3352 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3353 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3357 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3358 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3359 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3360 		if (rdev->pdev->device == 0x9648)  in evergreen_gpu_init()
3361 			rdev->config.evergreen.max_simds = 3;  in evergreen_gpu_init()
3362 		else if ((rdev->pdev->device == 0x9647) ||  in evergreen_gpu_init()
3363 			 (rdev->pdev->device == 0x964a))  in evergreen_gpu_init()
3364 			rdev->config.evergreen.max_simds = 4;  in evergreen_gpu_init()
3366 			rdev->config.evergreen.max_simds = 5;  in evergreen_gpu_init()
3367 		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3368 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3369 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3370 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3371 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3372 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3373 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3374 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3375 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3376 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3377 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3379 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3380 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3381 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3385 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3386 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3387 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3388 		rdev->config.evergreen.max_simds = 2;  in evergreen_gpu_init()
3389 		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3390 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3391 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3392 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3393 		rdev->config.evergreen.max_stack_entries = 512;  in evergreen_gpu_init()
3394 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3395 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3396 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3397 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3398 		rdev->config.evergreen.max_hw_contexts = 4;  in evergreen_gpu_init()
3399 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3401 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3402 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3403 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3407 		rdev->config.evergreen.num_ses = 2;  in evergreen_gpu_init()
3408 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3409 		rdev->config.evergreen.max_tile_pipes = 8;  in evergreen_gpu_init()
3410 		rdev->config.evergreen.max_simds = 7;  in evergreen_gpu_init()
3411 		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3412 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3413 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3414 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3415 		rdev->config.evergreen.max_stack_entries = 512;  in evergreen_gpu_init()
3416 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3417 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3418 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3419 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3420 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3421 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3423 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3424 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3425 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3429 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3430 		rdev->config.evergreen.max_pipes = 4;  in evergreen_gpu_init()
3431 		rdev->config.evergreen.max_tile_pipes = 4;  in evergreen_gpu_init()
3432 		rdev->config.evergreen.max_simds = 6;  in evergreen_gpu_init()
3433 		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3434 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3435 		rdev->config.evergreen.max_threads = 248;  in evergreen_gpu_init()
3436 		rdev->config.evergreen.max_gs_threads = 32;  in evergreen_gpu_init()
3437 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3438 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3439 		rdev->config.evergreen.sx_max_export_size = 256;  in evergreen_gpu_init()
3440 		rdev->config.evergreen.sx_max_export_pos_size = 64;  in evergreen_gpu_init()
3441 		rdev->config.evergreen.sx_max_export_smx_size = 192;  in evergreen_gpu_init()
3442 		rdev->config.evergreen.max_hw_contexts = 8;  in evergreen_gpu_init()
3443 		rdev->config.evergreen.sq_num_cf_insts = 2;  in evergreen_gpu_init()
3445 		rdev->config.evergreen.sc_prim_fifo_size = 0x100;  in evergreen_gpu_init()
3446 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3447 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3451 		rdev->config.evergreen.num_ses = 1;  in evergreen_gpu_init()
3452 		rdev->config.evergreen.max_pipes = 2;  in evergreen_gpu_init()
3453 		rdev->config.evergreen.max_tile_pipes = 2;  in evergreen_gpu_init()
3454 		rdev->config.evergreen.max_simds = 2;  in evergreen_gpu_init()
3455 		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;  in evergreen_gpu_init()
3456 		rdev->config.evergreen.max_gprs = 256;  in evergreen_gpu_init()
3457 		rdev->config.evergreen.max_threads = 192;  in evergreen_gpu_init()
3458 		rdev->config.evergreen.max_gs_threads = 16;  in evergreen_gpu_init()
3459 		rdev->config.evergreen.max_stack_entries = 256;  in evergreen_gpu_init()
3460 		rdev->config.evergreen.sx_num_of_sets = 4;  in evergreen_gpu_init()
3461 		rdev->config.evergreen.sx_max_export_size = 128;  in evergreen_gpu_init()
3462 		rdev->config.evergreen.sx_max_export_pos_size = 32;  in evergreen_gpu_init()
3463 		rdev->config.evergreen.sx_max_export_smx_size = 96;  in evergreen_gpu_init()
3464 		rdev->config.evergreen.max_hw_contexts = 4;  in evergreen_gpu_init()
3465 		rdev->config.evergreen.sq_num_cf_insts = 1;  in evergreen_gpu_init()
3467 		rdev->config.evergreen.sc_prim_fifo_size = 0x40;  in evergreen_gpu_init()
3468 		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;  in evergreen_gpu_init()
3469 		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;  in evergreen_gpu_init()
3487 	evergreen_fix_pci_max_read_req_size(rdev);  in evergreen_gpu_init()
3490 	if ((rdev->family == CHIP_PALM) ||  in evergreen_gpu_init()
3491 	    (rdev->family == CHIP_SUMO) ||  in evergreen_gpu_init()
3492 	    (rdev->family == CHIP_SUMO2))  in evergreen_gpu_init()
3504 	rdev->config.evergreen.tile_config = 0;  in evergreen_gpu_init()
3505 	switch (rdev->config.evergreen.max_tile_pipes) {  in evergreen_gpu_init()
3508 		rdev->config.evergreen.tile_config |= (0 << 0);  in evergreen_gpu_init()
3511 		rdev->config.evergreen.tile_config |= (1 << 0);  in evergreen_gpu_init()
3514 		rdev->config.evergreen.tile_config |= (2 << 0);  in evergreen_gpu_init()
3517 		rdev->config.evergreen.tile_config |= (3 << 0);  in evergreen_gpu_init()
3521 	if (rdev->flags & RADEON_IS_IGP)  in evergreen_gpu_init()
3522 		rdev->config.evergreen.tile_config |= 1 << 4;  in evergreen_gpu_init()
3526 			rdev->config.evergreen.tile_config |= 0 << 4;  in evergreen_gpu_init()
3529 			rdev->config.evergreen.tile_config |= 1 << 4;  in evergreen_gpu_init()
3533 			rdev->config.evergreen.tile_config |= 2 << 4;  in evergreen_gpu_init()
3537 	rdev->config.evergreen.tile_config |= 0 << 8;  in evergreen_gpu_init()
3538 	rdev->config.evergreen.tile_config |=  in evergreen_gpu_init()
3541 	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {  in evergreen_gpu_init()
3551 		for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {  in evergreen_gpu_init()
3564 	for (i = 0; i < rdev->config.evergreen.max_backends; i++)  in evergreen_gpu_init()
3568 		for (i = 0; i < rdev->config.evergreen.max_backends; i++)  in evergreen_gpu_init()
3572 	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {  in evergreen_gpu_init()
3578 		simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;  in evergreen_gpu_init()
3582 	rdev->config.evergreen.active_simds = hweight32(~tmp);  in evergreen_gpu_init()
3595 	if ((rdev->config.evergreen.max_backends == 1) &&  in evergreen_gpu_init()
3596 	    (rdev->flags & RADEON_IS_IGP)) {  in evergreen_gpu_init()
3606 		tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,  in evergreen_gpu_init()
3634 	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);  in evergreen_gpu_init()
3637 	if (rdev->family <= CHIP_SUMO2)  in evergreen_gpu_init()
3640 …WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) …  in evergreen_gpu_init()
3641 					POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |  in evergreen_gpu_init()
3642 					SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));  in evergreen_gpu_init()
3644 	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |  in evergreen_gpu_init()
3645 				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |  in evergreen_gpu_init()
3646 				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));  in evergreen_gpu_init()
3653 	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |  in evergreen_gpu_init()
3670 	switch (rdev->family) {  in evergreen_gpu_init()
3685 	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);  in evergreen_gpu_init()
3686 	sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);  in evergreen_gpu_init()
3688 	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);  in evergreen_gpu_init()
3689 	sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);  in evergreen_gpu_init()
3690 	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);  in evergreen_gpu_init()
3691 	sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);  in evergreen_gpu_init()
3693 	switch (rdev->family) {  in evergreen_gpu_init()
3706 …sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count)…  in evergreen_gpu_init()
3707 …sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count)…  in evergreen_gpu_init()
3708 …sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count)…  in evergreen_gpu_init()
3709 …sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count…  in evergreen_gpu_init()
3710 …sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_coun…  in evergreen_gpu_init()
3712 …sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6…  in evergreen_gpu_init()
3713 …sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / …  in evergreen_gpu_init()
3714 …sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6…  in evergreen_gpu_init()
3715 …sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / …  in evergreen_gpu_init()
3716 …sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6…  in evergreen_gpu_init()
3717 …sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / …  in evergreen_gpu_init()
3734 	switch (rdev->family) {  in evergreen_gpu_init()
3798 int evergreen_mc_init(struct radeon_device *rdev)  in evergreen_mc_init()  argument
3804 	rdev->mc.vram_is_ddr = true;  in evergreen_mc_init()
3805 	if ((rdev->family == CHIP_PALM) ||  in evergreen_mc_init()
3806 	    (rdev->family == CHIP_SUMO) ||  in evergreen_mc_init()
3807 	    (rdev->family == CHIP_SUMO2))  in evergreen_mc_init()
3834 	rdev->mc.vram_width = numchan * chansize;  in evergreen_mc_init()
3836 	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);  in evergreen_mc_init()
3837 	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);  in evergreen_mc_init()
3839 	if ((rdev->family == CHIP_PALM) ||  in evergreen_mc_init()
3840 	    (rdev->family == CHIP_SUMO) ||  in evergreen_mc_init()
3841 	    (rdev->family == CHIP_SUMO2)) {  in evergreen_mc_init()
3843 		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);  in evergreen_mc_init()
3844 		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);  in evergreen_mc_init()
3847 		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in evergreen_mc_init()
3848 		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in evergreen_mc_init()
3850 	rdev->mc.visible_vram_size = rdev->mc.aper_size;  in evergreen_mc_init()
3851 	r700_vram_gtt_location(rdev, &rdev->mc);  in evergreen_mc_init()
3852 	radeon_update_bandwidth_info(rdev);  in evergreen_mc_init()
3857 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)  in evergreen_print_gpu_status_regs()  argument
3859 	dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3861 	dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3863 	dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3865 	dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3867 	dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3869 	dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3871 	dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3873 	dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3875 	dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3877 	dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3879 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_print_gpu_status_regs()
3880 		dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",  in evergreen_print_gpu_status_regs()
3885 bool evergreen_is_display_hung(struct radeon_device *rdev)  in evergreen_is_display_hung()  argument
3891 	for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_is_display_hung()
3899 		for (i = 0; i < rdev->num_crtc; i++) {  in evergreen_is_display_hung()
3914 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)  in evergreen_gpu_check_soft_reset()  argument
3966 	if (evergreen_is_display_hung(rdev))  in evergreen_gpu_check_soft_reset()
3983 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)  in evergreen_gpu_soft_reset()  argument
3992 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);  in evergreen_gpu_soft_reset()
3994 	evergreen_print_gpu_status_regs(rdev);  in evergreen_gpu_soft_reset()
4008 	evergreen_mc_stop(rdev, &save);  in evergreen_gpu_soft_reset()
4009 	if (evergreen_mc_wait_for_idle(rdev)) {  in evergreen_gpu_soft_reset()
4010 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in evergreen_gpu_soft_reset()
4055 	if (!(rdev->flags & RADEON_IS_IGP)) {  in evergreen_gpu_soft_reset()
4063 		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);  in evergreen_gpu_soft_reset()
4077 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in evergreen_gpu_soft_reset()
4091 	evergreen_mc_resume(rdev, &save);  in evergreen_gpu_soft_reset()
4094 	evergreen_print_gpu_status_regs(rdev);  in evergreen_gpu_soft_reset()
4097 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)  in evergreen_gpu_pci_config_reset()  argument
4102 	dev_info(rdev->dev, "GPU pci config reset\n");  in evergreen_gpu_pci_config_reset()
4116 	r600_rlc_stop(rdev);  in evergreen_gpu_pci_config_reset()
4121 	rv770_set_clk_bypass_mode(rdev);  in evergreen_gpu_pci_config_reset()
4123 	pci_clear_master(rdev->pdev);  in evergreen_gpu_pci_config_reset()
4125 	evergreen_mc_stop(rdev, &save);  in evergreen_gpu_pci_config_reset()
4126 	if (evergreen_mc_wait_for_idle(rdev)) {  in evergreen_gpu_pci_config_reset()
4127 		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");  in evergreen_gpu_pci_config_reset()
4130 	radeon_pci_config_reset(rdev);  in evergreen_gpu_pci_config_reset()
4132 	for (i = 0; i < rdev->usec_timeout; i++) {  in evergreen_gpu_pci_config_reset()
4139 int evergreen_asic_reset(struct radeon_device *rdev)  in evergreen_asic_reset()  argument
4143 	reset_mask = evergreen_gpu_check_soft_reset(rdev);  in evergreen_asic_reset()
4146 		r600_set_bios_scratch_engine_hung(rdev, true);  in evergreen_asic_reset()
4149 	evergreen_gpu_soft_reset(rdev, reset_mask);  in evergreen_asic_reset()
4151 	reset_mask = evergreen_gpu_check_soft_reset(rdev);  in evergreen_asic_reset()
4155 		evergreen_gpu_pci_config_reset(rdev);  in evergreen_asic_reset()
4157 	reset_mask = evergreen_gpu_check_soft_reset(rdev);  in evergreen_asic_reset()
4160 		r600_set_bios_scratch_engine_hung(rdev, false);  in evergreen_asic_reset()
4174 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in evergreen_gfx_is_lockup()  argument
4176 	u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);  in evergreen_gfx_is_lockup()
4181 		radeon_ring_lockup_update(rdev, ring);  in evergreen_gfx_is_lockup()
4184 	return radeon_ring_test_lockup(rdev, ring);  in evergreen_gfx_is_lockup()
4193 void sumo_rlc_fini(struct radeon_device *rdev)  in sumo_rlc_fini()  argument
4198 	if (rdev->rlc.save_restore_obj) {  in sumo_rlc_fini()
4199 		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);  in sumo_rlc_fini()
4201 			dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);  in sumo_rlc_fini()
4202 		radeon_bo_unpin(rdev->rlc.save_restore_obj);  in sumo_rlc_fini()
4203 		radeon_bo_unreserve(rdev->rlc.save_restore_obj);  in sumo_rlc_fini()
4205 		radeon_bo_unref(&rdev->rlc.save_restore_obj);  in sumo_rlc_fini()
4206 		rdev->rlc.save_restore_obj = NULL;  in sumo_rlc_fini()
4210 	if (rdev->rlc.clear_state_obj) {  in sumo_rlc_fini()
4211 		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);  in sumo_rlc_fini()
4213 			dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);  in sumo_rlc_fini()
4214 		radeon_bo_unpin(rdev->rlc.clear_state_obj);  in sumo_rlc_fini()
4215 		radeon_bo_unreserve(rdev->rlc.clear_state_obj);  in sumo_rlc_fini()
4217 		radeon_bo_unref(&rdev->rlc.clear_state_obj);  in sumo_rlc_fini()
4218 		rdev->rlc.clear_state_obj = NULL;  in sumo_rlc_fini()
4222 	if (rdev->rlc.cp_table_obj) {  in sumo_rlc_fini()
4223 		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);  in sumo_rlc_fini()
4225 			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);  in sumo_rlc_fini()
4226 		radeon_bo_unpin(rdev->rlc.cp_table_obj);  in sumo_rlc_fini()
4227 		radeon_bo_unreserve(rdev->rlc.cp_table_obj);  in sumo_rlc_fini()
4229 		radeon_bo_unref(&rdev->rlc.cp_table_obj);  in sumo_rlc_fini()
4230 		rdev->rlc.cp_table_obj = NULL;  in sumo_rlc_fini()
4236 int sumo_rlc_init(struct radeon_device *rdev)  in sumo_rlc_init()  argument
4246 	src_ptr = rdev->rlc.reg_list;  in sumo_rlc_init()
4247 	dws = rdev->rlc.reg_list_size;  in sumo_rlc_init()
4248 	if (rdev->family >= CHIP_BONAIRE) {  in sumo_rlc_init()
4251 	cs_data = rdev->rlc.cs_data;  in sumo_rlc_init()
4255 		if (rdev->rlc.save_restore_obj == NULL) {  in sumo_rlc_init()
4256 			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,  in sumo_rlc_init()
4258 					     NULL, &rdev->rlc.save_restore_obj);  in sumo_rlc_init()
4260 				dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);  in sumo_rlc_init()
4265 		r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);  in sumo_rlc_init()
4267 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4270 		r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,  in sumo_rlc_init()
4271 				  &rdev->rlc.save_restore_gpu_addr);  in sumo_rlc_init()
4273 			radeon_bo_unreserve(rdev->rlc.save_restore_obj);  in sumo_rlc_init()
4274 			dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);  in sumo_rlc_init()
4275 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4279 		r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);  in sumo_rlc_init()
4281 			dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);  in sumo_rlc_init()
4282 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4286 		dst_ptr = rdev->rlc.sr_ptr;  in sumo_rlc_init()
4287 		if (rdev->family >= CHIP_TAHITI) {  in sumo_rlc_init()
4289 			for (i = 0; i < rdev->rlc.reg_list_size; i++)  in sumo_rlc_init()
4309 		radeon_bo_kunmap(rdev->rlc.save_restore_obj);  in sumo_rlc_init()
4310 		radeon_bo_unreserve(rdev->rlc.save_restore_obj);  in sumo_rlc_init()
4315 		if (rdev->family >= CHIP_BONAIRE) {  in sumo_rlc_init()
4316 			rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);  in sumo_rlc_init()
4317 		} else if (rdev->family >= CHIP_TAHITI) {  in sumo_rlc_init()
4318 			rdev->rlc.clear_state_size = si_get_csb_size(rdev);  in sumo_rlc_init()
4319 			dws = rdev->rlc.clear_state_size + (256 / 4);  in sumo_rlc_init()
4331 			rdev->rlc.clear_state_size = dws;  in sumo_rlc_init()
4334 		if (rdev->rlc.clear_state_obj == NULL) {  in sumo_rlc_init()
4335 			r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,  in sumo_rlc_init()
4337 					     NULL, &rdev->rlc.clear_state_obj);  in sumo_rlc_init()
4339 				dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);  in sumo_rlc_init()
4340 				sumo_rlc_fini(rdev);  in sumo_rlc_init()
4344 		r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);  in sumo_rlc_init()
4346 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4349 		r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,  in sumo_rlc_init()
4350 				  &rdev->rlc.clear_state_gpu_addr);  in sumo_rlc_init()
4352 			radeon_bo_unreserve(rdev->rlc.clear_state_obj);  in sumo_rlc_init()
4353 			dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);  in sumo_rlc_init()
4354 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4358 		r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);  in sumo_rlc_init()
4360 			dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);  in sumo_rlc_init()
4361 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4365 		dst_ptr = rdev->rlc.cs_ptr;  in sumo_rlc_init()
4366 		if (rdev->family >= CHIP_BONAIRE) {  in sumo_rlc_init()
4367 			cik_get_csb_buffer(rdev, dst_ptr);  in sumo_rlc_init()
4368 		} else if (rdev->family >= CHIP_TAHITI) {  in sumo_rlc_init()
4369 			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;  in sumo_rlc_init()
4372 			dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);  in sumo_rlc_init()
4373 			si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);  in sumo_rlc_init()
4376 			reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);  in sumo_rlc_init()
4405 		radeon_bo_kunmap(rdev->rlc.clear_state_obj);  in sumo_rlc_init()
4406 		radeon_bo_unreserve(rdev->rlc.clear_state_obj);  in sumo_rlc_init()
4409 	if (rdev->rlc.cp_table_size) {  in sumo_rlc_init()
4410 		if (rdev->rlc.cp_table_obj == NULL) {  in sumo_rlc_init()
4411 			r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,  in sumo_rlc_init()
4414 					     NULL, &rdev->rlc.cp_table_obj);  in sumo_rlc_init()
4416 				dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);  in sumo_rlc_init()
4417 				sumo_rlc_fini(rdev);  in sumo_rlc_init()
4422 		r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);  in sumo_rlc_init()
4424 			dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);  in sumo_rlc_init()
4425 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4428 		r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,  in sumo_rlc_init()
4429 				  &rdev->rlc.cp_table_gpu_addr);  in sumo_rlc_init()
4431 			radeon_bo_unreserve(rdev->rlc.cp_table_obj);  in sumo_rlc_init()
4432 			dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);  in sumo_rlc_init()
4433 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4436 		r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);  in sumo_rlc_init()
4438 			dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);  in sumo_rlc_init()
4439 			sumo_rlc_fini(rdev);  in sumo_rlc_init()
4443 		cik_init_cp_pg_table(rdev);  in sumo_rlc_init()
4445 		radeon_bo_kunmap(rdev->rlc.cp_table_obj);  in sumo_rlc_init()
4446 		radeon_bo_unreserve(rdev->rlc.cp_table_obj);  in sumo_rlc_init()
4453 static void evergreen_rlc_start(struct radeon_device *rdev)  in evergreen_rlc_start()  argument
4457 	if (rdev->flags & RADEON_IS_IGP) {  in evergreen_rlc_start()
4464 int evergreen_rlc_resume(struct radeon_device *rdev)  in evergreen_rlc_resume()  argument
4469 	if (!rdev->rlc_fw)  in evergreen_rlc_resume()
4472 	r600_rlc_stop(rdev);  in evergreen_rlc_resume()
4476 	if (rdev->flags & RADEON_IS_IGP) {  in evergreen_rlc_resume()
4477 		if (rdev->family == CHIP_ARUBA) {  in evergreen_rlc_resume()
4479 				3 | (3 << (16 * rdev->config.cayman.max_shader_engines));  in evergreen_rlc_resume()
4482 			tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;  in evergreen_rlc_resume()
4484 			if (tmp == rdev->config.cayman.max_simds_per_se) {  in evergreen_rlc_resume()
4495 		WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);  in evergreen_rlc_resume()
4496 		WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);  in evergreen_rlc_resume()
4507 	fw_data = (const __be32 *)rdev->rlc_fw->data;  in evergreen_rlc_resume()
4508 	if (rdev->family >= CHIP_ARUBA) {  in evergreen_rlc_resume()
4513 	} else if (rdev->family >= CHIP_CAYMAN) {  in evergreen_rlc_resume()
4526 	evergreen_rlc_start(rdev);  in evergreen_rlc_resume()
4533 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)  in evergreen_get_vblank_counter()  argument
4535 	if (crtc >= rdev->num_crtc)  in evergreen_get_vblank_counter()
4541 void evergreen_disable_interrupt_state(struct radeon_device *rdev)  in evergreen_disable_interrupt_state()  argument
4545 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_disable_interrupt_state()
4546 		cayman_cp_int_cntl_setup(rdev, 0,  in evergreen_disable_interrupt_state()
4548 		cayman_cp_int_cntl_setup(rdev, 1, 0);  in evergreen_disable_interrupt_state()
4549 		cayman_cp_int_cntl_setup(rdev, 2, 0);  in evergreen_disable_interrupt_state()
4560 	if (rdev->num_crtc >= 4) {  in evergreen_disable_interrupt_state()
4564 	if (rdev->num_crtc >= 6) {  in evergreen_disable_interrupt_state()
4571 	if (rdev->num_crtc >= 4) {  in evergreen_disable_interrupt_state()
4575 	if (rdev->num_crtc >= 6) {  in evergreen_disable_interrupt_state()
4581 	if (!ASIC_IS_DCE5(rdev))  in evergreen_disable_interrupt_state()
4600 int evergreen_irq_set(struct radeon_device *rdev)  in evergreen_irq_set()  argument
4611 	if (!rdev->irq.installed) {  in evergreen_irq_set()
4616 	if (!rdev->ih.enabled) {  in evergreen_irq_set()
4617 		r600_disable_interrupts(rdev);  in evergreen_irq_set()
4619 		evergreen_disable_interrupt_state(rdev);  in evergreen_irq_set()
4629 	if (rdev->family == CHIP_ARUBA)  in evergreen_irq_set()
4645 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_set()
4647 		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {  in evergreen_irq_set()
4651 		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {  in evergreen_irq_set()
4655 		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {  in evergreen_irq_set()
4660 		if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {  in evergreen_irq_set()
4667 	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {  in evergreen_irq_set()
4672 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_set()
4674 		if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {  in evergreen_irq_set()
4680 	if (rdev->irq.dpm_thermal) {  in evergreen_irq_set()
4685 	if (rdev->irq.crtc_vblank_int[0] ||  in evergreen_irq_set()
4686 	    atomic_read(&rdev->irq.pflip[0])) {  in evergreen_irq_set()
4690 	if (rdev->irq.crtc_vblank_int[1] ||  in evergreen_irq_set()
4691 	    atomic_read(&rdev->irq.pflip[1])) {  in evergreen_irq_set()
4695 	if (rdev->irq.crtc_vblank_int[2] ||  in evergreen_irq_set()
4696 	    atomic_read(&rdev->irq.pflip[2])) {  in evergreen_irq_set()
4700 	if (rdev->irq.crtc_vblank_int[3] ||  in evergreen_irq_set()
4701 	    atomic_read(&rdev->irq.pflip[3])) {  in evergreen_irq_set()
4705 	if (rdev->irq.crtc_vblank_int[4] ||  in evergreen_irq_set()
4706 	    atomic_read(&rdev->irq.pflip[4])) {  in evergreen_irq_set()
4710 	if (rdev->irq.crtc_vblank_int[5] ||  in evergreen_irq_set()
4711 	    atomic_read(&rdev->irq.pflip[5])) {  in evergreen_irq_set()
4715 	if (rdev->irq.hpd[0]) {  in evergreen_irq_set()
4719 	if (rdev->irq.hpd[1]) {  in evergreen_irq_set()
4723 	if (rdev->irq.hpd[2]) {  in evergreen_irq_set()
4727 	if (rdev->irq.hpd[3]) {  in evergreen_irq_set()
4731 	if (rdev->irq.hpd[4]) {  in evergreen_irq_set()
4735 	if (rdev->irq.hpd[5]) {  in evergreen_irq_set()
4739 	if (rdev->irq.afmt[0]) {  in evergreen_irq_set()
4743 	if (rdev->irq.afmt[1]) {  in evergreen_irq_set()
4747 	if (rdev->irq.afmt[2]) {  in evergreen_irq_set()
4751 	if (rdev->irq.afmt[3]) {  in evergreen_irq_set()
4755 	if (rdev->irq.afmt[4]) {  in evergreen_irq_set()
4759 	if (rdev->irq.afmt[5]) {  in evergreen_irq_set()
4764 	if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_set()
4765 		cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);  in evergreen_irq_set()
4766 		cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);  in evergreen_irq_set()
4767 		cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);  in evergreen_irq_set()
4773 	if (rdev->family >= CHIP_CAYMAN)  in evergreen_irq_set()
4780 	if (rdev->num_crtc >= 4) {  in evergreen_irq_set()
4784 	if (rdev->num_crtc >= 6) {  in evergreen_irq_set()
4793 	if (rdev->num_crtc >= 4) {  in evergreen_irq_set()
4799 	if (rdev->num_crtc >= 6) {  in evergreen_irq_set()
4812 	if (rdev->family == CHIP_ARUBA)  in evergreen_irq_set()
4830 static void evergreen_irq_ack(struct radeon_device *rdev)  in evergreen_irq_ack()  argument
4834 	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);  in evergreen_irq_ack()
4835 	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);  in evergreen_irq_ack()
4836 	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);  in evergreen_irq_ack()
4837 	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);  in evergreen_irq_ack()
4838 	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);  in evergreen_irq_ack()
4839 	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);  in evergreen_irq_ack()
4840 …rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSE…  in evergreen_irq_ack()
4841 …rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSE…  in evergreen_irq_ack()
4842 	if (rdev->num_crtc >= 4) {  in evergreen_irq_ack()
4843 …rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSE…  in evergreen_irq_ack()
4844 …rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSE…  in evergreen_irq_ack()
4846 	if (rdev->num_crtc >= 6) {  in evergreen_irq_ack()
4847 …rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSE…  in evergreen_irq_ack()
4848 …rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSE…  in evergreen_irq_ack()
4851 	rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);  in evergreen_irq_ack()
4852 	rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);  in evergreen_irq_ack()
4853 	rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);  in evergreen_irq_ack()
4854 	rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);  in evergreen_irq_ack()
4855 	rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);  in evergreen_irq_ack()
4856 	rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);  in evergreen_irq_ack()
4858 	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)  in evergreen_irq_ack()
4860 	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)  in evergreen_irq_ack()
4862 	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)  in evergreen_irq_ack()
4864 	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)  in evergreen_irq_ack()
4866 	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)  in evergreen_irq_ack()
4868 	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)  in evergreen_irq_ack()
4871 	if (rdev->num_crtc >= 4) {  in evergreen_irq_ack()
4872 		if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)  in evergreen_irq_ack()
4874 		if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)  in evergreen_irq_ack()
4876 		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)  in evergreen_irq_ack()
4878 		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)  in evergreen_irq_ack()
4880 		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)  in evergreen_irq_ack()
4882 		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)  in evergreen_irq_ack()
4886 	if (rdev->num_crtc >= 6) {  in evergreen_irq_ack()
4887 		if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)  in evergreen_irq_ack()
4889 		if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)  in evergreen_irq_ack()
4891 		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)  in evergreen_irq_ack()
4893 		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)  in evergreen_irq_ack()
4895 		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)  in evergreen_irq_ack()
4897 		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)  in evergreen_irq_ack()
4901 	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {  in evergreen_irq_ack()
4906 	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {  in evergreen_irq_ack()
4911 	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {  in evergreen_irq_ack()
4916 	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {  in evergreen_irq_ack()
4921 	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {  in evergreen_irq_ack()
4926 	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {  in evergreen_irq_ack()
4932 	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {  in evergreen_irq_ack()
4937 	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {  in evergreen_irq_ack()
4942 	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {  in evergreen_irq_ack()
4947 	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {  in evergreen_irq_ack()
4952 	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {  in evergreen_irq_ack()
4957 	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {  in evergreen_irq_ack()
4963 	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {  in evergreen_irq_ack()
4968 	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {  in evergreen_irq_ack()
4973 	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {  in evergreen_irq_ack()
4978 	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {  in evergreen_irq_ack()
4983 	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {  in evergreen_irq_ack()
4988 	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {  in evergreen_irq_ack()
4995 static void evergreen_irq_disable(struct radeon_device *rdev)  in evergreen_irq_disable()  argument
4997 	r600_disable_interrupts(rdev);  in evergreen_irq_disable()
5000 	evergreen_irq_ack(rdev);  in evergreen_irq_disable()
5001 	evergreen_disable_interrupt_state(rdev);  in evergreen_irq_disable()
5004 void evergreen_irq_suspend(struct radeon_device *rdev)  in evergreen_irq_suspend()  argument
5006 	evergreen_irq_disable(rdev);  in evergreen_irq_suspend()
5007 	r600_rlc_stop(rdev);  in evergreen_irq_suspend()
5010 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)  in evergreen_get_ih_wptr()  argument
5014 	if (rdev->wb.enabled)  in evergreen_get_ih_wptr()
5015 		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);  in evergreen_get_ih_wptr()
5025 		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",  in evergreen_get_ih_wptr()
5026 			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);  in evergreen_get_ih_wptr()
5027 		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;  in evergreen_get_ih_wptr()
5032 	return (wptr & rdev->ih.ptr_mask);  in evergreen_get_ih_wptr()
5035 int evergreen_irq_process(struct radeon_device *rdev)  in evergreen_irq_process()  argument
5047 	if (!rdev->ih.enabled || rdev->shutdown)  in evergreen_irq_process()
5050 	wptr = evergreen_get_ih_wptr(rdev);  in evergreen_irq_process()
5054 	if (atomic_xchg(&rdev->ih.lock, 1))  in evergreen_irq_process()
5057 	rptr = rdev->ih.rptr;  in evergreen_irq_process()
5064 	evergreen_irq_ack(rdev);  in evergreen_irq_process()
5069 		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;  in evergreen_irq_process()
5070 		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;  in evergreen_irq_process()
5076 				if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))  in evergreen_irq_process()
5079 				if (rdev->irq.crtc_vblank_int[0]) {  in evergreen_irq_process()
5080 					drm_handle_vblank(rdev->ddev, 0);  in evergreen_irq_process()
5081 					rdev->pm.vblank_sync = true;  in evergreen_irq_process()
5082 					wake_up(&rdev->irq.vblank_queue);  in evergreen_irq_process()
5084 				if (atomic_read(&rdev->irq.pflip[0]))  in evergreen_irq_process()
5085 					radeon_crtc_handle_vblank(rdev, 0);  in evergreen_irq_process()
5086 				rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;  in evergreen_irq_process()
5091 				if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))  in evergreen_irq_process()
5094 				rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;  in evergreen_irq_process()
5106 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))  in evergreen_irq_process()
5109 				if (rdev->irq.crtc_vblank_int[1]) {  in evergreen_irq_process()
5110 					drm_handle_vblank(rdev->ddev, 1);  in evergreen_irq_process()
5111 					rdev->pm.vblank_sync = true;  in evergreen_irq_process()
5112 					wake_up(&rdev->irq.vblank_queue);  in evergreen_irq_process()
5114 				if (atomic_read(&rdev->irq.pflip[1]))  in evergreen_irq_process()
5115 					radeon_crtc_handle_vblank(rdev, 1);  in evergreen_irq_process()
5116 				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;  in evergreen_irq_process()
5121 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))  in evergreen_irq_process()
5124 				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;  in evergreen_irq_process()
5136 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))  in evergreen_irq_process()
5139 				if (rdev->irq.crtc_vblank_int[2]) {  in evergreen_irq_process()
5140 					drm_handle_vblank(rdev->ddev, 2);  in evergreen_irq_process()
5141 					rdev->pm.vblank_sync = true;  in evergreen_irq_process()
5142 					wake_up(&rdev->irq.vblank_queue);  in evergreen_irq_process()
5144 				if (atomic_read(&rdev->irq.pflip[2]))  in evergreen_irq_process()
5145 					radeon_crtc_handle_vblank(rdev, 2);  in evergreen_irq_process()
5146 				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;  in evergreen_irq_process()
5151 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))  in evergreen_irq_process()
5154 				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;  in evergreen_irq_process()
5166 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))  in evergreen_irq_process()
5169 				if (rdev->irq.crtc_vblank_int[3]) {  in evergreen_irq_process()
5170 					drm_handle_vblank(rdev->ddev, 3);  in evergreen_irq_process()
5171 					rdev->pm.vblank_sync = true;  in evergreen_irq_process()
5172 					wake_up(&rdev->irq.vblank_queue);  in evergreen_irq_process()
5174 				if (atomic_read(&rdev->irq.pflip[3]))  in evergreen_irq_process()
5175 					radeon_crtc_handle_vblank(rdev, 3);  in evergreen_irq_process()
5176 				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;  in evergreen_irq_process()
5181 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))  in evergreen_irq_process()
5184 				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;  in evergreen_irq_process()
5196 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))  in evergreen_irq_process()
5199 				if (rdev->irq.crtc_vblank_int[4]) {  in evergreen_irq_process()
5200 					drm_handle_vblank(rdev->ddev, 4);  in evergreen_irq_process()
5201 					rdev->pm.vblank_sync = true;  in evergreen_irq_process()
5202 					wake_up(&rdev->irq.vblank_queue);  in evergreen_irq_process()
5204 				if (atomic_read(&rdev->irq.pflip[4]))  in evergreen_irq_process()
5205 					radeon_crtc_handle_vblank(rdev, 4);  in evergreen_irq_process()
5206 				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;  in evergreen_irq_process()
5211 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))  in evergreen_irq_process()
5214 				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;  in evergreen_irq_process()
5226 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))  in evergreen_irq_process()
5229 				if (rdev->irq.crtc_vblank_int[5]) {  in evergreen_irq_process()
5230 					drm_handle_vblank(rdev->ddev, 5);  in evergreen_irq_process()
5231 					rdev->pm.vblank_sync = true;  in evergreen_irq_process()
5232 					wake_up(&rdev->irq.vblank_queue);  in evergreen_irq_process()
5234 				if (atomic_read(&rdev->irq.pflip[5]))  in evergreen_irq_process()
5235 					radeon_crtc_handle_vblank(rdev, 5);  in evergreen_irq_process()
5236 				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;  in evergreen_irq_process()
5241 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))  in evergreen_irq_process()
5244 				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;  in evergreen_irq_process()
5261 				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);  in evergreen_irq_process()
5266 				if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))  in evergreen_irq_process()
5269 				rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;  in evergreen_irq_process()
5274 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))  in evergreen_irq_process()
5277 				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;  in evergreen_irq_process()
5282 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))  in evergreen_irq_process()
5285 				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;  in evergreen_irq_process()
5290 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))  in evergreen_irq_process()
5293 				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;  in evergreen_irq_process()
5298 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))  in evergreen_irq_process()
5301 				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;  in evergreen_irq_process()
5306 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))  in evergreen_irq_process()
5309 				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;  in evergreen_irq_process()
5314 				if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))  in evergreen_irq_process()
5317 				rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;  in evergreen_irq_process()
5322 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))  in evergreen_irq_process()
5325 				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;  in evergreen_irq_process()
5330 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))  in evergreen_irq_process()
5333 				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;  in evergreen_irq_process()
5338 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))  in evergreen_irq_process()
5341 				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;  in evergreen_irq_process()
5346 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))  in evergreen_irq_process()
5349 				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;  in evergreen_irq_process()
5354 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))  in evergreen_irq_process()
5357 				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;  in evergreen_irq_process()
5369 				if (!(rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG))  in evergreen_irq_process()
5372 				rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;  in evergreen_irq_process()
5377 				if (!(rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG))  in evergreen_irq_process()
5380 				rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;  in evergreen_irq_process()
5385 				if (!(rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG))  in evergreen_irq_process()
5388 				rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;  in evergreen_irq_process()
5393 				if (!(rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG))  in evergreen_irq_process()
5396 				rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;  in evergreen_irq_process()
5401 				if (!(rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG))  in evergreen_irq_process()
5404 				rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;  in evergreen_irq_process()
5409 				if (!(rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG))  in evergreen_irq_process()
5412 				rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;  in evergreen_irq_process()
5426 			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);  in evergreen_irq_process()
5436 			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);  in evergreen_irq_process()
5437 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in evergreen_irq_process()
5439 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in evergreen_irq_process()
5441 			cayman_vm_decode_fault(rdev, status, addr);  in evergreen_irq_process()
5447 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in evergreen_irq_process()
5451 			if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_process()
5454 					radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in evergreen_irq_process()
5457 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);  in evergreen_irq_process()
5460 					radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);  in evergreen_irq_process()
5464 				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);  in evergreen_irq_process()
5468 			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);  in evergreen_irq_process()
5472 			rdev->pm.dpm.thermal.high_to_low = false;  in evergreen_irq_process()
5477 			rdev->pm.dpm.thermal.high_to_low = true;  in evergreen_irq_process()
5484 			if (rdev->family >= CHIP_CAYMAN) {  in evergreen_irq_process()
5486 				radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);  in evergreen_irq_process()
5496 		rptr &= rdev->ih.ptr_mask;  in evergreen_irq_process()
5500 		schedule_work(&rdev->dp_work);  in evergreen_irq_process()
5502 		schedule_delayed_work(&rdev->hotplug_work, 0);  in evergreen_irq_process()
5504 		schedule_work(&rdev->audio_work);  in evergreen_irq_process()
5505 	if (queue_thermal && rdev->pm.dpm_enabled)  in evergreen_irq_process()
5506 		schedule_work(&rdev->pm.dpm.thermal.work);  in evergreen_irq_process()
5507 	rdev->ih.rptr = rptr;  in evergreen_irq_process()
5508 	atomic_set(&rdev->ih.lock, 0);  in evergreen_irq_process()
5511 	wptr = evergreen_get_ih_wptr(rdev);  in evergreen_irq_process()
5518 static int evergreen_startup(struct radeon_device *rdev)  in evergreen_startup()  argument
5524 	evergreen_pcie_gen2_enable(rdev);  in evergreen_startup()
5526 	evergreen_program_aspm(rdev);  in evergreen_startup()
5529 	r = r600_vram_scratch_init(rdev);  in evergreen_startup()
5533 	evergreen_mc_program(rdev);  in evergreen_startup()
5535 	if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {  in evergreen_startup()
5536 		r = ni_mc_load_microcode(rdev);  in evergreen_startup()
5543 	if (rdev->flags & RADEON_IS_AGP) {  in evergreen_startup()
5544 		evergreen_agp_enable(rdev);  in evergreen_startup()
5546 		r = evergreen_pcie_gart_enable(rdev);  in evergreen_startup()
5550 	evergreen_gpu_init(rdev);  in evergreen_startup()
5553 	if (rdev->flags & RADEON_IS_IGP) {  in evergreen_startup()
5554 		rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;  in evergreen_startup()
5555 		rdev->rlc.reg_list_size =  in evergreen_startup()
5557 		rdev->rlc.cs_data = evergreen_cs_data;  in evergreen_startup()
5558 		r = sumo_rlc_init(rdev);  in evergreen_startup()
5566 	r = radeon_wb_init(rdev);  in evergreen_startup()
5570 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);  in evergreen_startup()
5572 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in evergreen_startup()
5576 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);  in evergreen_startup()
5578 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in evergreen_startup()
5582 	r = uvd_v2_2_resume(rdev);  in evergreen_startup()
5584 		r = radeon_fence_driver_start_ring(rdev,  in evergreen_startup()
5587 			dev_err(rdev->dev, "UVD fences init error (%d).\n", r);  in evergreen_startup()
5591 		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;  in evergreen_startup()
5594 	if (!rdev->irq.installed) {  in evergreen_startup()
5595 		r = radeon_irq_kms_init(rdev);  in evergreen_startup()
5600 	r = r600_irq_init(rdev);  in evergreen_startup()
5603 		radeon_irq_kms_fini(rdev);  in evergreen_startup()
5606 	evergreen_irq_set(rdev);  in evergreen_startup()
5608 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in evergreen_startup()
5609 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,  in evergreen_startup()
5614 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in evergreen_startup()
5615 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,  in evergreen_startup()
5620 	r = evergreen_cp_load_microcode(rdev);  in evergreen_startup()
5623 	r = evergreen_cp_resume(rdev);  in evergreen_startup()
5626 	r = r600_dma_resume(rdev);  in evergreen_startup()
5630 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];  in evergreen_startup()
5632 		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,  in evergreen_startup()
5635 			r = uvd_v1_0_init(rdev);  in evergreen_startup()
5641 	r = radeon_ib_pool_init(rdev);  in evergreen_startup()
5643 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);  in evergreen_startup()
5647 	r = radeon_audio_init(rdev);  in evergreen_startup()
5656 int evergreen_resume(struct radeon_device *rdev)  in evergreen_resume()  argument
5663 	if (radeon_asic_reset(rdev))  in evergreen_resume()
5664 		dev_warn(rdev->dev, "GPU reset failed !\n");  in evergreen_resume()
5670 	atom_asic_init(rdev->mode_info.atom_context);  in evergreen_resume()
5673 	evergreen_init_golden_registers(rdev);  in evergreen_resume()
5675 	if (rdev->pm.pm_method == PM_METHOD_DPM)  in evergreen_resume()
5676 		radeon_pm_resume(rdev);  in evergreen_resume()
5678 	rdev->accel_working = true;  in evergreen_resume()
5679 	r = evergreen_startup(rdev);  in evergreen_resume()
5682 		rdev->accel_working = false;  in evergreen_resume()
5690 int evergreen_suspend(struct radeon_device *rdev)  in evergreen_suspend()  argument
5692 	radeon_pm_suspend(rdev);  in evergreen_suspend()
5693 	radeon_audio_fini(rdev);  in evergreen_suspend()
5694 	uvd_v1_0_fini(rdev);  in evergreen_suspend()
5695 	radeon_uvd_suspend(rdev);  in evergreen_suspend()
5696 	r700_cp_stop(rdev);  in evergreen_suspend()
5697 	r600_dma_stop(rdev);  in evergreen_suspend()
5698 	evergreen_irq_suspend(rdev);  in evergreen_suspend()
5699 	radeon_wb_disable(rdev);  in evergreen_suspend()
5700 	evergreen_pcie_gart_disable(rdev);  in evergreen_suspend()
5711 int evergreen_init(struct radeon_device *rdev)  in evergreen_init()  argument
5716 	if (!radeon_get_bios(rdev)) {  in evergreen_init()
5717 		if (ASIC_IS_AVIVO(rdev))  in evergreen_init()
5721 	if (!rdev->is_atom_bios) {  in evergreen_init()
5722 		dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");  in evergreen_init()
5725 	r = radeon_atombios_init(rdev);  in evergreen_init()
5731 	if (radeon_asic_reset(rdev))  in evergreen_init()
5732 		dev_warn(rdev->dev, "GPU reset failed !\n");  in evergreen_init()
5734 	if (!radeon_card_posted(rdev)) {  in evergreen_init()
5735 		if (!rdev->bios) {  in evergreen_init()
5736 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");  in evergreen_init()
5740 		atom_asic_init(rdev->mode_info.atom_context);  in evergreen_init()
5743 	evergreen_init_golden_registers(rdev);  in evergreen_init()
5745 	r600_scratch_init(rdev);  in evergreen_init()
5747 	radeon_surface_init(rdev);  in evergreen_init()
5749 	radeon_get_clock_info(rdev->ddev);  in evergreen_init()
5751 	r = radeon_fence_driver_init(rdev);  in evergreen_init()
5755 	if (rdev->flags & RADEON_IS_AGP) {  in evergreen_init()
5756 		r = radeon_agp_init(rdev);  in evergreen_init()
5758 			radeon_agp_disable(rdev);  in evergreen_init()
5761 	r = evergreen_mc_init(rdev);  in evergreen_init()
5765 	r = radeon_bo_init(rdev);  in evergreen_init()
5769 	if (ASIC_IS_DCE5(rdev)) {  in evergreen_init()
5770 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {  in evergreen_init()
5771 			r = ni_init_microcode(rdev);  in evergreen_init()
5778 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {  in evergreen_init()
5779 			r = r600_init_microcode(rdev);  in evergreen_init()
5788 	radeon_pm_init(rdev);  in evergreen_init()
5790 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;  in evergreen_init()
5791 	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);  in evergreen_init()
5793 	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;  in evergreen_init()
5794 	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);  in evergreen_init()
5796 	r = radeon_uvd_init(rdev);  in evergreen_init()
5798 		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;  in evergreen_init()
5799 		r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],  in evergreen_init()
5803 	rdev->ih.ring_obj = NULL;  in evergreen_init()
5804 	r600_ih_ring_init(rdev, 64 * 1024);  in evergreen_init()
5806 	r = r600_pcie_gart_init(rdev);  in evergreen_init()
5810 	rdev->accel_working = true;  in evergreen_init()
5811 	r = evergreen_startup(rdev);  in evergreen_init()
5813 		dev_err(rdev->dev, "disabling GPU acceleration\n");  in evergreen_init()
5814 		r700_cp_fini(rdev);  in evergreen_init()
5815 		r600_dma_fini(rdev);  in evergreen_init()
5816 		r600_irq_fini(rdev);  in evergreen_init()
5817 		if (rdev->flags & RADEON_IS_IGP)  in evergreen_init()
5818 			sumo_rlc_fini(rdev);  in evergreen_init()
5819 		radeon_wb_fini(rdev);  in evergreen_init()
5820 		radeon_ib_pool_fini(rdev);  in evergreen_init()
5821 		radeon_irq_kms_fini(rdev);  in evergreen_init()
5822 		evergreen_pcie_gart_fini(rdev);  in evergreen_init()
5823 		rdev->accel_working = false;  in evergreen_init()
5830 	if (ASIC_IS_DCE5(rdev)) {  in evergreen_init()
5831 		if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {  in evergreen_init()
5840 void evergreen_fini(struct radeon_device *rdev)  in evergreen_fini()  argument
5842 	radeon_pm_fini(rdev);  in evergreen_fini()
5843 	radeon_audio_fini(rdev);  in evergreen_fini()
5844 	r700_cp_fini(rdev);  in evergreen_fini()
5845 	r600_dma_fini(rdev);  in evergreen_fini()
5846 	r600_irq_fini(rdev);  in evergreen_fini()
5847 	if (rdev->flags & RADEON_IS_IGP)  in evergreen_fini()
5848 		sumo_rlc_fini(rdev);  in evergreen_fini()
5849 	radeon_wb_fini(rdev);  in evergreen_fini()
5850 	radeon_ib_pool_fini(rdev);  in evergreen_fini()
5851 	radeon_irq_kms_fini(rdev);  in evergreen_fini()
5852 	uvd_v1_0_fini(rdev);  in evergreen_fini()
5853 	radeon_uvd_fini(rdev);  in evergreen_fini()
5854 	evergreen_pcie_gart_fini(rdev);  in evergreen_fini()
5855 	r600_vram_scratch_fini(rdev);  in evergreen_fini()
5856 	radeon_gem_fini(rdev);  in evergreen_fini()
5857 	radeon_fence_driver_fini(rdev);  in evergreen_fini()
5858 	radeon_agp_fini(rdev);  in evergreen_fini()
5859 	radeon_bo_fini(rdev);  in evergreen_fini()
5860 	radeon_atombios_fini(rdev);  in evergreen_fini()
5861 	kfree(rdev->bios);  in evergreen_fini()
5862 	rdev->bios = NULL;  in evergreen_fini()
5865 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)  in evergreen_pcie_gen2_enable()  argument
5872 	if (rdev->flags & RADEON_IS_IGP)  in evergreen_pcie_gen2_enable()
5875 	if (!(rdev->flags & RADEON_IS_PCIE))  in evergreen_pcie_gen2_enable()
5879 	if (ASIC_IS_X2(rdev))  in evergreen_pcie_gen2_enable()
5882 	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&  in evergreen_pcie_gen2_enable()
5883 		(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))  in evergreen_pcie_gen2_enable()
5928 void evergreen_program_aspm(struct radeon_device *rdev)  in evergreen_program_aspm()  argument
5943 	if (!(rdev->flags & RADEON_IS_PCIE))  in evergreen_program_aspm()
5946 	switch (rdev->family) {  in evergreen_program_aspm()
5963 	if (rdev->flags & RADEON_IS_IGP)  in evergreen_program_aspm()
5985 		if (rdev->family >= CHIP_BARTS)  in evergreen_program_aspm()
5992 		if (rdev->family >= CHIP_BARTS)  in evergreen_program_aspm()
6022 			if (rdev->family >= CHIP_BARTS) {  in evergreen_program_aspm()
6054 			if (rdev->family >= CHIP_BARTS) {  in evergreen_program_aspm()
6071 	if (rdev->family < CHIP_BARTS)  in evergreen_program_aspm()