165 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
166 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
167 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
168 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
169 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
170 extern void evergreen_mc_program(struct radeon_device *rdev);
171 extern void evergreen_irq_suspend(struct radeon_device *rdev);
172 extern int evergreen_mc_init(struct radeon_device *rdev);
173 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
174 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
175 extern void evergreen_program_aspm(struct radeon_device *rdev);
176 extern void sumo_rlc_fini(struct radeon_device *rdev);
177 extern int sumo_rlc_init(struct radeon_device *rdev);
178 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
428 static void ni_init_golden_registers(struct radeon_device *rdev)  in ni_init_golden_registers()  argument
430 	switch (rdev->family) {  in ni_init_golden_registers()
432 		radeon_program_register_sequence(rdev,  in ni_init_golden_registers()
435 		radeon_program_register_sequence(rdev,  in ni_init_golden_registers()
440 		if ((rdev->pdev->device == 0x9900) ||  in ni_init_golden_registers()
441 		    (rdev->pdev->device == 0x9901) ||  in ni_init_golden_registers()
442 		    (rdev->pdev->device == 0x9903) ||  in ni_init_golden_registers()
443 		    (rdev->pdev->device == 0x9904) ||  in ni_init_golden_registers()
444 		    (rdev->pdev->device == 0x9905) ||  in ni_init_golden_registers()
445 		    (rdev->pdev->device == 0x9906) ||  in ni_init_golden_registers()
446 		    (rdev->pdev->device == 0x9907) ||  in ni_init_golden_registers()
447 		    (rdev->pdev->device == 0x9908) ||  in ni_init_golden_registers()
448 		    (rdev->pdev->device == 0x9909) ||  in ni_init_golden_registers()
449 		    (rdev->pdev->device == 0x990A) ||  in ni_init_golden_registers()
450 		    (rdev->pdev->device == 0x990B) ||  in ni_init_golden_registers()
451 		    (rdev->pdev->device == 0x990C) ||  in ni_init_golden_registers()
452 		    (rdev->pdev->device == 0x990D) ||  in ni_init_golden_registers()
453 		    (rdev->pdev->device == 0x990E) ||  in ni_init_golden_registers()
454 		    (rdev->pdev->device == 0x990F) ||  in ni_init_golden_registers()
455 		    (rdev->pdev->device == 0x9910) ||  in ni_init_golden_registers()
456 		    (rdev->pdev->device == 0x9913) ||  in ni_init_golden_registers()
457 		    (rdev->pdev->device == 0x9917) ||  in ni_init_golden_registers()
458 		    (rdev->pdev->device == 0x9918)) {  in ni_init_golden_registers()
459 			radeon_program_register_sequence(rdev,  in ni_init_golden_registers()
462 			radeon_program_register_sequence(rdev,  in ni_init_golden_registers()
466 			radeon_program_register_sequence(rdev,  in ni_init_golden_registers()
469 			radeon_program_register_sequence(rdev,  in ni_init_golden_registers()
609 int ni_mc_load_microcode(struct radeon_device *rdev)  in ni_mc_load_microcode()  argument
616 	if (!rdev->mc_fw)  in ni_mc_load_microcode()
619 	switch (rdev->family) {  in ni_mc_load_microcode()
662 		fw_data = (const __be32 *)rdev->mc_fw->data;  in ni_mc_load_microcode()
672 		for (i = 0; i < rdev->usec_timeout; i++) {  in ni_mc_load_microcode()
685 int ni_init_microcode(struct radeon_device *rdev)  in ni_init_microcode()  argument
696 	switch (rdev->family) {  in ni_init_microcode()
748 	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);  in ni_init_microcode()
751 	if (rdev->pfp_fw->size != pfp_req_size) {  in ni_init_microcode()
754 		       rdev->pfp_fw->size, fw_name);  in ni_init_microcode()
760 	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);  in ni_init_microcode()
763 	if (rdev->me_fw->size != me_req_size) {  in ni_init_microcode()
766 		       rdev->me_fw->size, fw_name);  in ni_init_microcode()
771 	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);  in ni_init_microcode()
774 	if (rdev->rlc_fw->size != rlc_req_size) {  in ni_init_microcode()
777 		       rdev->rlc_fw->size, fw_name);  in ni_init_microcode()
782 	if (!(rdev->flags & RADEON_IS_IGP)) {  in ni_init_microcode()
784 		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in ni_init_microcode()
787 		if (rdev->mc_fw->size != mc_req_size) {  in ni_init_microcode()
790 			       rdev->mc_fw->size, fw_name);  in ni_init_microcode()
795 	if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {  in ni_init_microcode()
797 		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);  in ni_init_microcode()
802 			release_firmware(rdev->smc_fw);  in ni_init_microcode()
803 			rdev->smc_fw = NULL;  in ni_init_microcode()
805 		} else if (rdev->smc_fw->size != smc_req_size) {  in ni_init_microcode()
808 			       rdev->mc_fw->size, fw_name);  in ni_init_microcode()
819 		release_firmware(rdev->pfp_fw);  in ni_init_microcode()
820 		rdev->pfp_fw = NULL;  in ni_init_microcode()
821 		release_firmware(rdev->me_fw);  in ni_init_microcode()
822 		rdev->me_fw = NULL;  in ni_init_microcode()
823 		release_firmware(rdev->rlc_fw);  in ni_init_microcode()
824 		rdev->rlc_fw = NULL;  in ni_init_microcode()
825 		release_firmware(rdev->mc_fw);  in ni_init_microcode()
826 		rdev->mc_fw = NULL;  in ni_init_microcode()
841 int cayman_get_allowed_info_register(struct radeon_device *rdev,  in cayman_get_allowed_info_register()  argument
860 int tn_get_temp(struct radeon_device *rdev)  in tn_get_temp()  argument
871 static void cayman_gpu_init(struct radeon_device *rdev)  in cayman_gpu_init()  argument
884 	switch (rdev->family) {  in cayman_gpu_init()
886 		rdev->config.cayman.max_shader_engines = 2;  in cayman_gpu_init()
887 		rdev->config.cayman.max_pipes_per_simd = 4;  in cayman_gpu_init()
888 		rdev->config.cayman.max_tile_pipes = 8;  in cayman_gpu_init()
889 		rdev->config.cayman.max_simds_per_se = 12;  in cayman_gpu_init()
890 		rdev->config.cayman.max_backends_per_se = 4;  in cayman_gpu_init()
891 		rdev->config.cayman.max_texture_channel_caches = 8;  in cayman_gpu_init()
892 		rdev->config.cayman.max_gprs = 256;  in cayman_gpu_init()
893 		rdev->config.cayman.max_threads = 256;  in cayman_gpu_init()
894 		rdev->config.cayman.max_gs_threads = 32;  in cayman_gpu_init()
895 		rdev->config.cayman.max_stack_entries = 512;  in cayman_gpu_init()
896 		rdev->config.cayman.sx_num_of_sets = 8;  in cayman_gpu_init()
897 		rdev->config.cayman.sx_max_export_size = 256;  in cayman_gpu_init()
898 		rdev->config.cayman.sx_max_export_pos_size = 64;  in cayman_gpu_init()
899 		rdev->config.cayman.sx_max_export_smx_size = 192;  in cayman_gpu_init()
900 		rdev->config.cayman.max_hw_contexts = 8;  in cayman_gpu_init()
901 		rdev->config.cayman.sq_num_cf_insts = 2;  in cayman_gpu_init()
903 		rdev->config.cayman.sc_prim_fifo_size = 0x100;  in cayman_gpu_init()
904 		rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;  in cayman_gpu_init()
905 		rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;  in cayman_gpu_init()
910 		rdev->config.cayman.max_shader_engines = 1;  in cayman_gpu_init()
911 		rdev->config.cayman.max_pipes_per_simd = 4;  in cayman_gpu_init()
912 		rdev->config.cayman.max_tile_pipes = 2;  in cayman_gpu_init()
913 		if ((rdev->pdev->device == 0x9900) ||  in cayman_gpu_init()
914 		    (rdev->pdev->device == 0x9901) ||  in cayman_gpu_init()
915 		    (rdev->pdev->device == 0x9905) ||  in cayman_gpu_init()
916 		    (rdev->pdev->device == 0x9906) ||  in cayman_gpu_init()
917 		    (rdev->pdev->device == 0x9907) ||  in cayman_gpu_init()
918 		    (rdev->pdev->device == 0x9908) ||  in cayman_gpu_init()
919 		    (rdev->pdev->device == 0x9909) ||  in cayman_gpu_init()
920 		    (rdev->pdev->device == 0x990B) ||  in cayman_gpu_init()
921 		    (rdev->pdev->device == 0x990C) ||  in cayman_gpu_init()
922 		    (rdev->pdev->device == 0x990F) ||  in cayman_gpu_init()
923 		    (rdev->pdev->device == 0x9910) ||  in cayman_gpu_init()
924 		    (rdev->pdev->device == 0x9917) ||  in cayman_gpu_init()
925 		    (rdev->pdev->device == 0x9999) ||  in cayman_gpu_init()
926 		    (rdev->pdev->device == 0x999C)) {  in cayman_gpu_init()
927 			rdev->config.cayman.max_simds_per_se = 6;  in cayman_gpu_init()
928 			rdev->config.cayman.max_backends_per_se = 2;  in cayman_gpu_init()
929 			rdev->config.cayman.max_hw_contexts = 8;  in cayman_gpu_init()
930 			rdev->config.cayman.sx_max_export_size = 256;  in cayman_gpu_init()
931 			rdev->config.cayman.sx_max_export_pos_size = 64;  in cayman_gpu_init()
932 			rdev->config.cayman.sx_max_export_smx_size = 192;  in cayman_gpu_init()
933 		} else if ((rdev->pdev->device == 0x9903) ||  in cayman_gpu_init()
934 			   (rdev->pdev->device == 0x9904) ||  in cayman_gpu_init()
935 			   (rdev->pdev->device == 0x990A) ||  in cayman_gpu_init()
936 			   (rdev->pdev->device == 0x990D) ||  in cayman_gpu_init()
937 			   (rdev->pdev->device == 0x990E) ||  in cayman_gpu_init()
938 			   (rdev->pdev->device == 0x9913) ||  in cayman_gpu_init()
939 			   (rdev->pdev->device == 0x9918) ||  in cayman_gpu_init()
940 			   (rdev->pdev->device == 0x999D)) {  in cayman_gpu_init()
941 			rdev->config.cayman.max_simds_per_se = 4;  in cayman_gpu_init()
942 			rdev->config.cayman.max_backends_per_se = 2;  in cayman_gpu_init()
943 			rdev->config.cayman.max_hw_contexts = 8;  in cayman_gpu_init()
944 			rdev->config.cayman.sx_max_export_size = 256;  in cayman_gpu_init()
945 			rdev->config.cayman.sx_max_export_pos_size = 64;  in cayman_gpu_init()
946 			rdev->config.cayman.sx_max_export_smx_size = 192;  in cayman_gpu_init()
947 		} else if ((rdev->pdev->device == 0x9919) ||  in cayman_gpu_init()
948 			   (rdev->pdev->device == 0x9990) ||  in cayman_gpu_init()
949 			   (rdev->pdev->device == 0x9991) ||  in cayman_gpu_init()
950 			   (rdev->pdev->device == 0x9994) ||  in cayman_gpu_init()
951 			   (rdev->pdev->device == 0x9995) ||  in cayman_gpu_init()
952 			   (rdev->pdev->device == 0x9996) ||  in cayman_gpu_init()
953 			   (rdev->pdev->device == 0x999A) ||  in cayman_gpu_init()
954 			   (rdev->pdev->device == 0x99A0)) {  in cayman_gpu_init()
955 			rdev->config.cayman.max_simds_per_se = 3;  in cayman_gpu_init()
956 			rdev->config.cayman.max_backends_per_se = 1;  in cayman_gpu_init()
957 			rdev->config.cayman.max_hw_contexts = 4;  in cayman_gpu_init()
958 			rdev->config.cayman.sx_max_export_size = 128;  in cayman_gpu_init()
959 			rdev->config.cayman.sx_max_export_pos_size = 32;  in cayman_gpu_init()
960 			rdev->config.cayman.sx_max_export_smx_size = 96;  in cayman_gpu_init()
962 			rdev->config.cayman.max_simds_per_se = 2;  in cayman_gpu_init()
963 			rdev->config.cayman.max_backends_per_se = 1;  in cayman_gpu_init()
964 			rdev->config.cayman.max_hw_contexts = 4;  in cayman_gpu_init()
965 			rdev->config.cayman.sx_max_export_size = 128;  in cayman_gpu_init()
966 			rdev->config.cayman.sx_max_export_pos_size = 32;  in cayman_gpu_init()
967 			rdev->config.cayman.sx_max_export_smx_size = 96;  in cayman_gpu_init()
969 		rdev->config.cayman.max_texture_channel_caches = 2;  in cayman_gpu_init()
970 		rdev->config.cayman.max_gprs = 256;  in cayman_gpu_init()
971 		rdev->config.cayman.max_threads = 256;  in cayman_gpu_init()
972 		rdev->config.cayman.max_gs_threads = 32;  in cayman_gpu_init()
973 		rdev->config.cayman.max_stack_entries = 512;  in cayman_gpu_init()
974 		rdev->config.cayman.sx_num_of_sets = 8;  in cayman_gpu_init()
975 		rdev->config.cayman.sq_num_cf_insts = 2;  in cayman_gpu_init()
977 		rdev->config.cayman.sc_prim_fifo_size = 0x40;  in cayman_gpu_init()
978 		rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;  in cayman_gpu_init()
979 		rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;  in cayman_gpu_init()
997 	evergreen_fix_pci_max_read_req_size(rdev);  in cayman_gpu_init()
1003 	rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;  in cayman_gpu_init()
1004 	if (rdev->config.cayman.mem_row_size_in_kb > 4)  in cayman_gpu_init()
1005 		rdev->config.cayman.mem_row_size_in_kb = 4;  in cayman_gpu_init()
1007 	rdev->config.cayman.shader_engine_tile_size = 32;  in cayman_gpu_init()
1008 	rdev->config.cayman.num_gpus = 1;  in cayman_gpu_init()
1009 	rdev->config.cayman.multi_gpu_tile_size = 64;  in cayman_gpu_init()
1012 	rdev->config.cayman.num_tile_pipes = (1 << tmp);  in cayman_gpu_init()
1014 	rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;  in cayman_gpu_init()
1016 	rdev->config.cayman.num_shader_engines = tmp + 1;  in cayman_gpu_init()
1018 	rdev->config.cayman.num_gpus = tmp + 1;  in cayman_gpu_init()
1020 	rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;  in cayman_gpu_init()
1022 	rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;  in cayman_gpu_init()
1032 	rdev->config.cayman.tile_config = 0;  in cayman_gpu_init()
1033 	switch (rdev->config.cayman.num_tile_pipes) {  in cayman_gpu_init()
1036 		rdev->config.cayman.tile_config |= (0 << 0);  in cayman_gpu_init()
1039 		rdev->config.cayman.tile_config |= (1 << 0);  in cayman_gpu_init()
1042 		rdev->config.cayman.tile_config |= (2 << 0);  in cayman_gpu_init()
1045 		rdev->config.cayman.tile_config |= (3 << 0);  in cayman_gpu_init()
1050 	if (rdev->flags & RADEON_IS_IGP)  in cayman_gpu_init()
1051 		rdev->config.cayman.tile_config |= 1 << 4;  in cayman_gpu_init()
1055 			rdev->config.cayman.tile_config |= 0 << 4;  in cayman_gpu_init()
1058 			rdev->config.cayman.tile_config |= 1 << 4;  in cayman_gpu_init()
1062 			rdev->config.cayman.tile_config |= 2 << 4;  in cayman_gpu_init()
1066 	rdev->config.cayman.tile_config |=  in cayman_gpu_init()
1068 	rdev->config.cayman.tile_config |=  in cayman_gpu_init()
1072 	for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {  in cayman_gpu_init()
1084 …for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines)…  in cayman_gpu_init()
1088 …for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines)…  in cayman_gpu_init()
1092 	for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {  in cayman_gpu_init()
1098 		simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;  in cayman_gpu_init()
1102 	rdev->config.cayman.active_simds = hweight32(~tmp);  in cayman_gpu_init()
1109 	if (ASIC_IS_DCE6(rdev))  in cayman_gpu_init()
1118 	if ((rdev->config.cayman.max_backends_per_se == 1) &&  in cayman_gpu_init()
1119 	    (rdev->flags & RADEON_IS_IGP)) {  in cayman_gpu_init()
1129 		tmp = r6xx_remap_render_backend(rdev, tmp,  in cayman_gpu_init()
1130 						rdev->config.cayman.max_backends_per_se *  in cayman_gpu_init()
1131 						rdev->config.cayman.max_shader_engines,  in cayman_gpu_init()
1137 	for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)  in cayman_gpu_init()
1159 	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);  in cayman_gpu_init()
1175 …WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1…  in cayman_gpu_init()
1176 					POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |  in cayman_gpu_init()
1177 					SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));  in cayman_gpu_init()
1179 	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |  in cayman_gpu_init()
1180 				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |  in cayman_gpu_init()
1181 				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));  in cayman_gpu_init()
1188 	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |  in cayman_gpu_init()
1231 	if (rdev->family == CHIP_ARUBA) {  in cayman_gpu_init()
1244 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)  in cayman_pcie_gart_tlb_flush()  argument
1253 static int cayman_pcie_gart_enable(struct radeon_device *rdev)  in cayman_pcie_gart_enable()  argument
1257 	if (rdev->gart.robj == NULL) {  in cayman_pcie_gart_enable()
1258 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");  in cayman_pcie_gart_enable()
1261 	r = radeon_gart_table_vram_pin(rdev);  in cayman_pcie_gart_enable()
1284 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);  in cayman_pcie_gart_enable()
1285 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);  in cayman_pcie_gart_enable()
1286 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);  in cayman_pcie_gart_enable()
1288 			(u32)(rdev->dummy_page.addr >> 12));  in cayman_pcie_gart_enable()
1305 			rdev->vm_manager.max_pfn - 1);  in cayman_pcie_gart_enable()
1307 		       rdev->vm_manager.saved_table_addr[i]);  in cayman_pcie_gart_enable()
1312 	       (u32)(rdev->dummy_page.addr >> 12));  in cayman_pcie_gart_enable()
1329 	cayman_pcie_gart_tlb_flush(rdev);  in cayman_pcie_gart_enable()
1331 		 (unsigned)(rdev->mc.gtt_size >> 20),  in cayman_pcie_gart_enable()
1332 		 (unsigned long long)rdev->gart.table_addr);  in cayman_pcie_gart_enable()
1333 	rdev->gart.ready = true;  in cayman_pcie_gart_enable()
1337 static void cayman_pcie_gart_disable(struct radeon_device *rdev)  in cayman_pcie_gart_disable()  argument
1342 		rdev->vm_manager.saved_table_addr[i] = RREG32(  in cayman_pcie_gart_disable()
1361 	radeon_gart_table_vram_unpin(rdev);  in cayman_pcie_gart_disable()
1364 static void cayman_pcie_gart_fini(struct radeon_device *rdev)  in cayman_pcie_gart_fini()  argument
1366 	cayman_pcie_gart_disable(rdev);  in cayman_pcie_gart_fini()
1367 	radeon_gart_table_vram_free(rdev);  in cayman_pcie_gart_fini()
1368 	radeon_gart_fini(rdev);  in cayman_pcie_gart_fini()
1371 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,  in cayman_cp_int_cntl_setup()  argument
1383 void cayman_fence_ring_emit(struct radeon_device *rdev,  in cayman_fence_ring_emit()  argument
1386 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cayman_fence_ring_emit()
1387 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cayman_fence_ring_emit()
1406 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)  in cayman_ring_ib_execute()  argument
1408 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in cayman_ring_ib_execute()
1442 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)  in cayman_cp_enable()  argument
1447 		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in cayman_cp_enable()
1448 			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);  in cayman_cp_enable()
1451 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cayman_cp_enable()
1455 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,  in cayman_gfx_get_rptr()  argument
1460 	if (rdev->wb.enabled)  in cayman_gfx_get_rptr()
1461 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in cayman_gfx_get_rptr()
1474 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,  in cayman_gfx_get_wptr()  argument
1489 void cayman_gfx_set_wptr(struct radeon_device *rdev,  in cayman_gfx_set_wptr()  argument
1504 static int cayman_cp_load_microcode(struct radeon_device *rdev)  in cayman_cp_load_microcode()  argument
1509 	if (!rdev->me_fw || !rdev->pfp_fw)  in cayman_cp_load_microcode()
1512 	cayman_cp_enable(rdev, false);  in cayman_cp_load_microcode()
1514 	fw_data = (const __be32 *)rdev->pfp_fw->data;  in cayman_cp_load_microcode()
1520 	fw_data = (const __be32 *)rdev->me_fw->data;  in cayman_cp_load_microcode()
1531 static int cayman_cp_start(struct radeon_device *rdev)  in cayman_cp_start()  argument
1533 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cayman_cp_start()
1536 	r = radeon_ring_lock(rdev, ring, 7);  in cayman_cp_start()
1544 	radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);  in cayman_cp_start()
1548 	radeon_ring_unlock_commit(rdev, ring, false);  in cayman_cp_start()
1550 	cayman_cp_enable(rdev, true);  in cayman_cp_start()
1552 	r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);  in cayman_cp_start()
1590 	radeon_ring_unlock_commit(rdev, ring, false);  in cayman_cp_start()
1597 static void cayman_cp_fini(struct radeon_device *rdev)  in cayman_cp_fini()  argument
1599 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cayman_cp_fini()
1600 	cayman_cp_enable(rdev, false);  in cayman_cp_fini()
1601 	radeon_ring_fini(rdev, ring);  in cayman_cp_fini()
1602 	radeon_scratch_free(rdev, ring->rptr_save_reg);  in cayman_cp_fini()
1605 static int cayman_cp_resume(struct radeon_device *rdev)  in cayman_cp_resume()  argument
1666 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);  in cayman_cp_resume()
1674 		ring = &rdev->ring[ridx[i]];  in cayman_cp_resume()
1683 		addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;  in cayman_cp_resume()
1690 		ring = &rdev->ring[ridx[i]];  in cayman_cp_resume()
1696 		ring = &rdev->ring[ridx[i]];  in cayman_cp_resume()
1708 	cayman_cp_start(rdev);  in cayman_cp_resume()
1709 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;  in cayman_cp_resume()
1710 	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;  in cayman_cp_resume()
1711 	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;  in cayman_cp_resume()
1713 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);  in cayman_cp_resume()
1715 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cayman_cp_resume()
1716 		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;  in cayman_cp_resume()
1717 		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;  in cayman_cp_resume()
1721 	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in cayman_cp_resume()
1722 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);  in cayman_cp_resume()
1727 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)  in cayman_gpu_check_soft_reset()  argument
1788 	if (evergreen_is_display_hung(rdev))  in cayman_gpu_check_soft_reset()
1805 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)  in cayman_gpu_soft_reset()  argument
1814 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);  in cayman_gpu_soft_reset()
1816 	evergreen_print_gpu_status_regs(rdev);  in cayman_gpu_soft_reset()
1817 	dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",  in cayman_gpu_soft_reset()
1819 	dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",  in cayman_gpu_soft_reset()
1821 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in cayman_gpu_soft_reset()
1823 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in cayman_gpu_soft_reset()
1845 	evergreen_mc_stop(rdev, &save);  in cayman_gpu_soft_reset()
1846 	if (evergreen_mc_wait_for_idle(rdev)) {  in cayman_gpu_soft_reset()
1847 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cayman_gpu_soft_reset()
1895 	if (!(rdev->flags & RADEON_IS_IGP)) {  in cayman_gpu_soft_reset()
1903 		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);  in cayman_gpu_soft_reset()
1917 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in cayman_gpu_soft_reset()
1931 	evergreen_mc_resume(rdev, &save);  in cayman_gpu_soft_reset()
1934 	evergreen_print_gpu_status_regs(rdev);  in cayman_gpu_soft_reset()
1937 int cayman_asic_reset(struct radeon_device *rdev)  in cayman_asic_reset()  argument
1941 	reset_mask = cayman_gpu_check_soft_reset(rdev);  in cayman_asic_reset()
1944 		r600_set_bios_scratch_engine_hung(rdev, true);  in cayman_asic_reset()
1946 	cayman_gpu_soft_reset(rdev, reset_mask);  in cayman_asic_reset()
1948 	reset_mask = cayman_gpu_check_soft_reset(rdev);  in cayman_asic_reset()
1951 		evergreen_gpu_pci_config_reset(rdev);  in cayman_asic_reset()
1953 	r600_set_bios_scratch_engine_hung(rdev, false);  in cayman_asic_reset()
1967 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in cayman_gfx_is_lockup()  argument
1969 	u32 reset_mask = cayman_gpu_check_soft_reset(rdev);  in cayman_gfx_is_lockup()
1974 		radeon_ring_lockup_update(rdev, ring);  in cayman_gfx_is_lockup()
1977 	return radeon_ring_test_lockup(rdev, ring);  in cayman_gfx_is_lockup()
1980 static int cayman_startup(struct radeon_device *rdev)  in cayman_startup()  argument
1982 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cayman_startup()
1986 	evergreen_pcie_gen2_enable(rdev);  in cayman_startup()
1988 	evergreen_program_aspm(rdev);  in cayman_startup()
1991 	r = r600_vram_scratch_init(rdev);  in cayman_startup()
1995 	evergreen_mc_program(rdev);  in cayman_startup()
1997 	if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {  in cayman_startup()
1998 		r = ni_mc_load_microcode(rdev);  in cayman_startup()
2005 	r = cayman_pcie_gart_enable(rdev);  in cayman_startup()
2008 	cayman_gpu_init(rdev);  in cayman_startup()
2011 	if (rdev->flags & RADEON_IS_IGP) {  in cayman_startup()
2012 		rdev->rlc.reg_list = tn_rlc_save_restore_register_list;  in cayman_startup()
2013 		rdev->rlc.reg_list_size =  in cayman_startup()
2015 		rdev->rlc.cs_data = cayman_cs_data;  in cayman_startup()
2016 		r = sumo_rlc_init(rdev);  in cayman_startup()
2024 	r = radeon_wb_init(rdev);  in cayman_startup()
2028 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);  in cayman_startup()
2030 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cayman_startup()
2034 	r = uvd_v2_2_resume(rdev);  in cayman_startup()
2036 		r = radeon_fence_driver_start_ring(rdev,  in cayman_startup()
2039 			dev_err(rdev->dev, "UVD fences init error (%d).\n", r);  in cayman_startup()
2042 		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;  in cayman_startup()
2044 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);  in cayman_startup()
2046 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cayman_startup()
2050 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);  in cayman_startup()
2052 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cayman_startup()
2056 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);  in cayman_startup()
2058 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in cayman_startup()
2062 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);  in cayman_startup()
2064 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in cayman_startup()
2069 	if (!rdev->irq.installed) {  in cayman_startup()
2070 		r = radeon_irq_kms_init(rdev);  in cayman_startup()
2075 	r = r600_irq_init(rdev);  in cayman_startup()
2078 		radeon_irq_kms_fini(rdev);  in cayman_startup()
2081 	evergreen_irq_set(rdev);  in cayman_startup()
2083 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,  in cayman_startup()
2088 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cayman_startup()
2089 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,  in cayman_startup()
2094 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cayman_startup()
2095 	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,  in cayman_startup()
2100 	r = cayman_cp_load_microcode(rdev);  in cayman_startup()
2103 	r = cayman_cp_resume(rdev);  in cayman_startup()
2107 	r = cayman_dma_resume(rdev);  in cayman_startup()
2111 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];  in cayman_startup()
2113 		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,  in cayman_startup()
2116 			r = uvd_v1_0_init(rdev);  in cayman_startup()
2121 	r = radeon_ib_pool_init(rdev);  in cayman_startup()
2123 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);  in cayman_startup()
2127 	r = radeon_vm_manager_init(rdev);  in cayman_startup()
2129 		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);  in cayman_startup()
2133 	r = radeon_audio_init(rdev);  in cayman_startup()
2140 int cayman_resume(struct radeon_device *rdev)  in cayman_resume()  argument
2149 	atom_asic_init(rdev->mode_info.atom_context);  in cayman_resume()
2152 	ni_init_golden_registers(rdev);  in cayman_resume()
2154 	if (rdev->pm.pm_method == PM_METHOD_DPM)  in cayman_resume()
2155 		radeon_pm_resume(rdev);  in cayman_resume()
2157 	rdev->accel_working = true;  in cayman_resume()
2158 	r = cayman_startup(rdev);  in cayman_resume()
2161 		rdev->accel_working = false;  in cayman_resume()
2167 int cayman_suspend(struct radeon_device *rdev)  in cayman_suspend()  argument
2169 	radeon_pm_suspend(rdev);  in cayman_suspend()
2170 	radeon_audio_fini(rdev);  in cayman_suspend()
2171 	radeon_vm_manager_fini(rdev);  in cayman_suspend()
2172 	cayman_cp_enable(rdev, false);  in cayman_suspend()
2173 	cayman_dma_stop(rdev);  in cayman_suspend()
2174 	uvd_v1_0_fini(rdev);  in cayman_suspend()
2175 	radeon_uvd_suspend(rdev);  in cayman_suspend()
2176 	evergreen_irq_suspend(rdev);  in cayman_suspend()
2177 	radeon_wb_disable(rdev);  in cayman_suspend()
2178 	cayman_pcie_gart_disable(rdev);  in cayman_suspend()
2188 int cayman_init(struct radeon_device *rdev)  in cayman_init()  argument
2190 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cayman_init()
2194 	if (!radeon_get_bios(rdev)) {  in cayman_init()
2195 		if (ASIC_IS_AVIVO(rdev))  in cayman_init()
2199 	if (!rdev->is_atom_bios) {  in cayman_init()
2200 		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");  in cayman_init()
2203 	r = radeon_atombios_init(rdev);  in cayman_init()
2208 	if (!radeon_card_posted(rdev)) {  in cayman_init()
2209 		if (!rdev->bios) {  in cayman_init()
2210 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");  in cayman_init()
2214 		atom_asic_init(rdev->mode_info.atom_context);  in cayman_init()
2217 	ni_init_golden_registers(rdev);  in cayman_init()
2219 	r600_scratch_init(rdev);  in cayman_init()
2221 	radeon_surface_init(rdev);  in cayman_init()
2223 	radeon_get_clock_info(rdev->ddev);  in cayman_init()
2225 	r = radeon_fence_driver_init(rdev);  in cayman_init()
2229 	r = evergreen_mc_init(rdev);  in cayman_init()
2233 	r = radeon_bo_init(rdev);  in cayman_init()
2237 	if (rdev->flags & RADEON_IS_IGP) {  in cayman_init()
2238 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {  in cayman_init()
2239 			r = ni_init_microcode(rdev);  in cayman_init()
2246 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {  in cayman_init()
2247 			r = ni_init_microcode(rdev);  in cayman_init()
2256 	radeon_pm_init(rdev);  in cayman_init()
2259 	r600_ring_init(rdev, ring, 1024 * 1024);  in cayman_init()
2261 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cayman_init()
2263 	r600_ring_init(rdev, ring, 64 * 1024);  in cayman_init()
2265 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cayman_init()
2267 	r600_ring_init(rdev, ring, 64 * 1024);  in cayman_init()
2269 	r = radeon_uvd_init(rdev);  in cayman_init()
2271 		ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];  in cayman_init()
2273 		r600_ring_init(rdev, ring, 4096);  in cayman_init()
2276 	rdev->ih.ring_obj = NULL;  in cayman_init()
2277 	r600_ih_ring_init(rdev, 64 * 1024);  in cayman_init()
2279 	r = r600_pcie_gart_init(rdev);  in cayman_init()
2283 	rdev->accel_working = true;  in cayman_init()
2284 	r = cayman_startup(rdev);  in cayman_init()
2286 		dev_err(rdev->dev, "disabling GPU acceleration\n");  in cayman_init()
2287 		cayman_cp_fini(rdev);  in cayman_init()
2288 		cayman_dma_fini(rdev);  in cayman_init()
2289 		r600_irq_fini(rdev);  in cayman_init()
2290 		if (rdev->flags & RADEON_IS_IGP)  in cayman_init()
2291 			sumo_rlc_fini(rdev);  in cayman_init()
2292 		radeon_wb_fini(rdev);  in cayman_init()
2293 		radeon_ib_pool_fini(rdev);  in cayman_init()
2294 		radeon_vm_manager_fini(rdev);  in cayman_init()
2295 		radeon_irq_kms_fini(rdev);  in cayman_init()
2296 		cayman_pcie_gart_fini(rdev);  in cayman_init()
2297 		rdev->accel_working = false;  in cayman_init()
2307 	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {  in cayman_init()
2315 void cayman_fini(struct radeon_device *rdev)  in cayman_fini()  argument
2317 	radeon_pm_fini(rdev);  in cayman_fini()
2318 	cayman_cp_fini(rdev);  in cayman_fini()
2319 	cayman_dma_fini(rdev);  in cayman_fini()
2320 	r600_irq_fini(rdev);  in cayman_fini()
2321 	if (rdev->flags & RADEON_IS_IGP)  in cayman_fini()
2322 		sumo_rlc_fini(rdev);  in cayman_fini()
2323 	radeon_wb_fini(rdev);  in cayman_fini()
2324 	radeon_vm_manager_fini(rdev);  in cayman_fini()
2325 	radeon_ib_pool_fini(rdev);  in cayman_fini()
2326 	radeon_irq_kms_fini(rdev);  in cayman_fini()
2327 	uvd_v1_0_fini(rdev);  in cayman_fini()
2328 	radeon_uvd_fini(rdev);  in cayman_fini()
2329 	cayman_pcie_gart_fini(rdev);  in cayman_fini()
2330 	r600_vram_scratch_fini(rdev);  in cayman_fini()
2331 	radeon_gem_fini(rdev);  in cayman_fini()
2332 	radeon_fence_driver_fini(rdev);  in cayman_fini()
2333 	radeon_bo_fini(rdev);  in cayman_fini()
2334 	radeon_atombios_fini(rdev);  in cayman_fini()
2335 	kfree(rdev->bios);  in cayman_fini()
2336 	rdev->bios = NULL;  in cayman_fini()
2342 int cayman_vm_init(struct radeon_device *rdev)  in cayman_vm_init()  argument
2345 	rdev->vm_manager.nvm = 8;  in cayman_vm_init()
2347 	if (rdev->flags & RADEON_IS_IGP) {  in cayman_vm_init()
2350 		rdev->vm_manager.vram_base_offset = tmp;  in cayman_vm_init()
2352 		rdev->vm_manager.vram_base_offset = 0;  in cayman_vm_init()
2356 void cayman_vm_fini(struct radeon_device *rdev)  in cayman_vm_fini()  argument
2369 void cayman_vm_decode_fault(struct radeon_device *rdev,  in cayman_vm_decode_fault()  argument
2529 void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,  in cayman_vm_flush()  argument