This source file includes following definitions.
- gfx10_kiq_set_resources
- gfx10_kiq_map_queues
- gfx10_kiq_unmap_queues
- gfx10_kiq_query_status
- gfx_v10_0_set_kiq_pm4_funcs
- gfx_v10_0_init_golden_registers
- gfx_v10_0_scratch_init
- gfx_v10_0_write_data_to_reg
- gfx_v10_0_wait_reg_mem
- gfx_v10_0_ring_test_ring
- gfx_v10_0_ring_test_ib
- gfx_v10_0_free_microcode
- gfx_v10_0_check_fw_write_wait
- gfx_v10_0_init_rlc_ext_microcode
- gfx_v10_0_check_gfxoff_flag
- gfx_v10_0_init_microcode
- gfx_v10_0_get_csb_size
- gfx_v10_0_get_csb_buffer
- gfx_v10_0_rlc_fini
- gfx_v10_0_rlc_init
- gfx_v10_0_csb_vram_pin
- gfx_v10_0_csb_vram_unpin
- gfx_v10_0_mec_fini
- gfx_v10_0_me_init
- gfx_v10_0_mec_init
- wave_read_ind
- wave_read_regs
- gfx_v10_0_read_wave_data
- gfx_v10_0_read_wave_sgprs
- gfx_v10_0_read_wave_vgprs
- gfx_v10_0_select_me_pipe_q
- gfx_v10_0_gpu_early_init
- gfx_v10_0_gfx_ring_init
- gfx_v10_0_compute_ring_init
- gfx_v10_0_sw_init
- gfx_v10_0_pfp_fini
- gfx_v10_0_ce_fini
- gfx_v10_0_me_fini
- gfx_v10_0_sw_fini
- gfx_v10_0_tiling_mode_table_init
- gfx_v10_0_select_se_sh
- gfx_v10_0_get_rb_active_bitmap
- gfx_v10_0_setup_rb
- gfx_v10_0_init_pa_sc_tile_steering_override
- gfx_v10_0_init_compute_vmid
- gfx_v10_0_init_gds_vmid
- gfx_v10_0_tcp_harvest
- gfx_v10_0_get_tcc_info
- gfx_v10_0_constants_init
- gfx_v10_0_enable_gui_idle_interrupt
- gfx_v10_0_init_csb
- gfx_v10_0_init_pg
- gfx_v10_0_rlc_stop
- gfx_v10_0_rlc_reset
- gfx_v10_0_rlc_smu_handshake_cntl
- gfx_v10_0_rlc_start
- gfx_v10_0_rlc_enable_srm
- gfx_v10_0_rlc_load_microcode
- gfx_v10_0_rlc_resume
- gfx_v10_0_parse_rlc_toc
- gfx_v10_0_calc_toc_total_size
- gfx_v10_0_rlc_backdoor_autoload_buffer_init
- gfx_v10_0_rlc_backdoor_autoload_buffer_fini
- gfx_v10_0_rlc_backdoor_autoload_copy_ucode
- gfx_v10_0_rlc_backdoor_autoload_copy_toc_ucode
- gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode
- gfx_v10_0_rlc_backdoor_autoload_copy_sdma_ucode
- gfx_v10_0_rlc_backdoor_autoload_enable
- gfx_v10_0_rlc_backdoor_autoload_config_me_cache
- gfx_v10_0_rlc_backdoor_autoload_config_ce_cache
- gfx_v10_0_rlc_backdoor_autoload_config_pfp_cache
- gfx_v10_0_rlc_backdoor_autoload_config_mec_cache
- gfx_v10_0_wait_for_rlc_autoload_complete
- gfx_v10_0_cp_gfx_enable
- gfx_v10_0_cp_gfx_load_pfp_microcode
- gfx_v10_0_cp_gfx_load_ce_microcode
- gfx_v10_0_cp_gfx_load_me_microcode
- gfx_v10_0_cp_gfx_load_microcode
- gfx_v10_0_cp_gfx_start
- gfx_v10_0_cp_gfx_switch_pipe
- gfx_v10_0_cp_gfx_set_doorbell
- gfx_v10_0_cp_gfx_resume
- gfx_v10_0_cp_compute_enable
- gfx_v10_0_cp_compute_load_microcode
- gfx_v10_0_kiq_setting
- gfx_v10_0_gfx_mqd_init
- gfx_v10_0_gfx_queue_init_register
- gfx_v10_0_gfx_init_queue
- gfx_v10_0_kiq_enable_kgq
- gfx_v10_0_cp_async_gfx_ring_resume
- gfx_v10_0_compute_mqd_init
- gfx_v10_0_kiq_init_register
- gfx_v10_0_kiq_init_queue
- gfx_v10_0_kcq_init_queue
- gfx_v10_0_kiq_resume
- gfx_v10_0_kcq_resume
- gfx_v10_0_cp_resume
- gfx_v10_0_cp_enable
- gfx_v10_0_check_grbm_cam_remapping
- gfx_v10_0_setup_grbm_cam_remapping
- gfx_v10_0_hw_init
- gfx_v10_0_kiq_disable_kgq
- gfx_v10_0_hw_fini
- gfx_v10_0_suspend
- gfx_v10_0_resume
- gfx_v10_0_is_idle
- gfx_v10_0_wait_for_idle
- gfx_v10_0_soft_reset
- gfx_v10_0_get_gpu_clock_counter
- gfx_v10_0_ring_emit_gds_switch
- gfx_v10_0_early_init
- gfx_v10_0_late_init
- gfx_v10_0_is_rlc_enabled
- gfx_v10_0_set_safe_mode
- gfx_v10_0_unset_safe_mode
- gfx_v10_0_update_medium_grain_clock_gating
- gfx_v10_0_update_3d_clock_gating
- gfx_v10_0_update_coarse_grain_clock_gating
- gfx_v10_0_update_gfx_clock_gating
- gfx_v10_0_set_powergating_state
- gfx_v10_0_set_clockgating_state
- gfx_v10_0_get_clockgating_state
- gfx_v10_0_ring_get_rptr_gfx
- gfx_v10_0_ring_get_wptr_gfx
- gfx_v10_0_ring_set_wptr_gfx
- gfx_v10_0_ring_get_rptr_compute
- gfx_v10_0_ring_get_wptr_compute
- gfx_v10_0_ring_set_wptr_compute
- gfx_v10_0_ring_emit_hdp_flush
- gfx_v10_0_ring_emit_ib_gfx
- gfx_v10_0_ring_emit_ib_compute
- gfx_v10_0_ring_emit_fence
- gfx_v10_0_ring_emit_pipeline_sync
- gfx_v10_0_ring_emit_vm_flush
- gfx_v10_0_ring_emit_fence_kiq
- gfx_v10_0_ring_emit_sb
- gfx_v10_0_ring_emit_cntxcntl
- gfx_v10_0_ring_emit_init_cond_exec
- gfx_v10_0_ring_emit_patch_cond_exec
- gfx_v10_0_ring_preempt_ib
- gfx_v10_0_ring_emit_ce_meta
- gfx_v10_0_ring_emit_de_meta
- gfx_v10_0_ring_emit_tmz
- gfx_v10_0_ring_emit_rreg
- gfx_v10_0_ring_emit_wreg
- gfx_v10_0_ring_emit_reg_wait
- gfx_v10_0_ring_emit_reg_write_reg_wait
- gfx_v10_0_set_gfx_eop_interrupt_state
- gfx_v10_0_set_compute_eop_interrupt_state
- gfx_v10_0_set_eop_interrupt_state
- gfx_v10_0_eop_irq
- gfx_v10_0_set_priv_reg_fault_state
- gfx_v10_0_set_priv_inst_fault_state
- gfx_v10_0_handle_priv_fault
- gfx_v10_0_priv_reg_irq
- gfx_v10_0_priv_inst_irq
- gfx_v10_0_kiq_set_interrupt_state
- gfx_v10_0_kiq_irq
- gfx_v10_0_set_ring_funcs
- gfx_v10_0_set_irq_funcs
- gfx_v10_0_set_rlc_funcs
- gfx_v10_0_set_gds_init
- gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh
- gfx_v10_0_get_wgp_active_bitmap_per_sh
- gfx_v10_0_get_cu_active_bitmap_per_sh
- gfx_v10_0_get_cu_info
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 #include <linux/delay.h>
25 #include <linux/kernel.h>
26 #include <linux/firmware.h>
27 #include <linux/module.h>
28 #include <linux/pci.h>
29 #include "amdgpu.h"
30 #include "amdgpu_gfx.h"
31 #include "amdgpu_psp.h"
32 #include "amdgpu_smu.h"
33 #include "nv.h"
34 #include "nvd.h"
35
36 #include "gc/gc_10_1_0_offset.h"
37 #include "gc/gc_10_1_0_sh_mask.h"
38 #include "navi10_enum.h"
39 #include "hdp/hdp_5_0_0_offset.h"
40 #include "ivsrcid/gfx/irqsrcs_gfx_10_1.h"
41
42 #include "soc15.h"
43 #include "soc15_common.h"
44 #include "clearstate_gfx10.h"
45 #include "v10_structs.h"
46 #include "gfx_v10_0.h"
47 #include "nbio_v2_3.h"
48
49
50
51
52
53
54
55
56
57 #define GFX10_NUM_GFX_RINGS 2
58 #define GFX10_MEC_HPD_SIZE 2048
59
60 #define F32_CE_PROGRAM_RAM_SIZE 65536
61 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
62
63 #define mmCGTT_GS_NGG_CLK_CTRL 0x5087
64 #define mmCGTT_GS_NGG_CLK_CTRL_BASE_IDX 1
65
66 MODULE_FIRMWARE("amdgpu/navi10_ce.bin");
67 MODULE_FIRMWARE("amdgpu/navi10_pfp.bin");
68 MODULE_FIRMWARE("amdgpu/navi10_me.bin");
69 MODULE_FIRMWARE("amdgpu/navi10_mec.bin");
70 MODULE_FIRMWARE("amdgpu/navi10_mec2.bin");
71 MODULE_FIRMWARE("amdgpu/navi10_rlc.bin");
72
73 MODULE_FIRMWARE("amdgpu/navi14_ce_wks.bin");
74 MODULE_FIRMWARE("amdgpu/navi14_pfp_wks.bin");
75 MODULE_FIRMWARE("amdgpu/navi14_me_wks.bin");
76 MODULE_FIRMWARE("amdgpu/navi14_mec_wks.bin");
77 MODULE_FIRMWARE("amdgpu/navi14_mec2_wks.bin");
78 MODULE_FIRMWARE("amdgpu/navi14_ce.bin");
79 MODULE_FIRMWARE("amdgpu/navi14_pfp.bin");
80 MODULE_FIRMWARE("amdgpu/navi14_me.bin");
81 MODULE_FIRMWARE("amdgpu/navi14_mec.bin");
82 MODULE_FIRMWARE("amdgpu/navi14_mec2.bin");
83 MODULE_FIRMWARE("amdgpu/navi14_rlc.bin");
84
85 MODULE_FIRMWARE("amdgpu/navi12_ce.bin");
86 MODULE_FIRMWARE("amdgpu/navi12_pfp.bin");
87 MODULE_FIRMWARE("amdgpu/navi12_me.bin");
88 MODULE_FIRMWARE("amdgpu/navi12_mec.bin");
89 MODULE_FIRMWARE("amdgpu/navi12_mec2.bin");
90 MODULE_FIRMWARE("amdgpu/navi12_rlc.bin");
91
92 static const struct soc15_reg_golden golden_settings_gc_10_1[] =
93 {
94 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_4, 0xffffffff, 0x00400014),
95 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_CPF_CLK_CTRL, 0xfcff8fff, 0xf8000100),
96 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SPI_CLK_CTRL, 0xcd000000, 0x0d000100),
97 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SQ_CLK_CTRL, 0x60000ff0, 0x60000100),
98 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SQG_CLK_CTRL, 0x40000000, 0x40000100),
99 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_VGT_CLK_CTRL, 0xffff8fff, 0xffff8100),
100 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_WD_CLK_CTRL, 0xfeff8fff, 0xfeff8100),
101 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCH_PIPE_STEER, 0xffffffff, 0xe4e4e4e4),
102 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCH_VC5_ENABLE, 0x00000002, 0x00000000),
103 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0x000007ff, 0x000005ff),
104 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG, 0x20000000, 0x20000000),
105 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xffffffff, 0x00000420),
106 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x00000200, 0x00000200),
107 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG4, 0x07900000, 0x04900000),
108 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DFSM_TILES_IN_FLIGHT, 0x0000ffff, 0x0000003f),
109 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_LAST_OF_BURST_CONFIG, 0xffffffff, 0x03860204),
110 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCR_GENERAL_CNTL, 0x1ff0ffff, 0x00000500),
111 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGE_PRIV_CONTROL, 0x000007ff, 0x000001fe),
112 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL1_PIPE_STEER, 0xffffffff, 0xe4e4e4e4),
113 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2_PIPE_STEER_0, 0x77777777, 0x10321032),
114 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2_PIPE_STEER_1, 0x77777777, 0x02310231),
115 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2A_ADDR_MATCH_MASK, 0xffffffff, 0xffffffcf),
116 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_ADDR_MATCH_MASK, 0xffffffff, 0xffffffcf),
117 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CGTT_SCLK_CTRL, 0x10000000, 0x10000100),
118 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CTRL2, 0xffffffff, 0x1402002f),
119 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CTRL3, 0xffff9fff, 0x00001188),
120 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x08000009),
121 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00400000, 0x04440000),
122 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
123 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_SPARE, 0xffffffff, 0xffff3101),
124 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_ALU_CLK_CTRL, 0xffffffff, 0xffffffff),
125 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_ARB_CONFIG, 0x00000100, 0x00000130),
126 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_LDS_CLK_CTRL, 0xffffffff, 0xffffffff),
127 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfff7ffff, 0x01030000),
128 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CNTL, 0x60000010, 0x479c0010),
129 SOC15_REG_GOLDEN_VALUE(GC, 0, mmUTCL1_CGTT_CLK_CTRL, 0xfeff0fff, 0x40000100),
130 SOC15_REG_GOLDEN_VALUE(GC, 0, mmUTCL1_CTRL, 0x00800000, 0x00800000)
131 };
132
133 static const struct soc15_reg_golden golden_settings_gc_10_0_nv10[] =
134 {
135
136 };
137
138 static const struct soc15_reg_golden golden_settings_gc_10_1_1[] =
139 {
140 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_4, 0xffffffff, 0x003c0014),
141 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_GS_NGG_CLK_CTRL, 0xffff8fff, 0xffff8100),
142 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_IA_CLK_CTRL, 0xffff0fff, 0xffff0100),
143 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SPI_CLK_CTRL, 0xcd000000, 0x0d000100),
144 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SQ_CLK_CTRL, 0xf8ff0fff, 0x60000100),
145 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SQG_CLK_CTRL, 0x40000ff0, 0x40000100),
146 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_VGT_CLK_CTRL, 0xffff8fff, 0xffff8100),
147 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_WD_CLK_CTRL, 0xffff8fff, 0xffff8100),
148 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCH_PIPE_STEER, 0xffffffff, 0xe4e4e4e4),
149 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCH_VC5_ENABLE, 0x00000002, 0x00000000),
150 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0x800007ff, 0x000005ff),
151 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG, 0xffffffff, 0x20000000),
152 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xffffffff, 0x00000420),
153 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x00000200, 0x00000200),
154 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG4, 0xffffffff, 0x04900000),
155 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DFSM_TILES_IN_FLIGHT, 0x0000ffff, 0x0000003f),
156 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_LAST_OF_BURST_CONFIG, 0xffffffff, 0x03860204),
157 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCR_GENERAL_CNTL, 0x1ff0ffff, 0x00000500),
158 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGE_PRIV_CONTROL, 0x000007ff, 0x000001fe),
159 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL1_PIPE_STEER, 0xffffffff, 0xe4e4e4e4),
160 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2A_ADDR_MATCH_MASK, 0xffffffff, 0xffffffe7),
161 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_ADDR_MATCH_MASK, 0xffffffff, 0xffffffe7),
162 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CGTT_SCLK_CTRL, 0xffff0fff, 0x10000100),
163 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CTRL2, 0xffffffff, 0x1402002f),
164 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CTRL3, 0xffffbfff, 0x00000188),
165 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x08000009),
166 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00400000, 0x04440000),
167 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
168 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_SPARE, 0xffffffff, 0xffff3101),
169 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_ALU_CLK_CTRL, 0xffffffff, 0xffffffff),
170 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_ARB_CONFIG, 0x00000133, 0x00000130),
171 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_LDS_CLK_CTRL, 0xffffffff, 0xffffffff),
172 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfff7ffff, 0x01030000),
173 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CNTL, 0x60000010, 0x479c0010),
174 SOC15_REG_GOLDEN_VALUE(GC, 0, mmUTCL1_CTRL, 0x00800000, 0x00800000),
175 };
176
177 static const struct soc15_reg_golden golden_settings_gc_10_1_2[] =
178 {
179 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_4, 0x003e001f, 0x003c0014),
180 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_GS_NGG_CLK_CTRL, 0xffff8fff, 0xffff8100),
181 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_IA_CLK_CTRL, 0xffff0fff, 0xffff0100),
182 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SPI_CLK_CTRL, 0xff7f0fff, 0x0d000100),
183 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SQ_CLK_CTRL, 0xffffcfff, 0x60000100),
184 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_SQG_CLK_CTRL, 0xffff0fff, 0x40000100),
185 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_VGT_CLK_CTRL, 0xffff8fff, 0xffff8100),
186 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCGTT_WD_CLK_CTRL, 0xffff8fff, 0xffff8100),
187 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCH_PIPE_STEER, 0xffffffff, 0xe4e4e4e4),
188 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCH_VC5_ENABLE, 0x00000003, 0x00000000),
189 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0x800007ff, 0x000005ff),
190 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG, 0xffffffff, 0x20000000),
191 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xffffffff, 0x00000420),
192 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0xffffffff, 0x00000200),
193 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG4, 0xffffffff, 0x04800000),
194 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DFSM_TILES_IN_FLIGHT, 0x0000ffff, 0x0000003f),
195 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_LAST_OF_BURST_CONFIG, 0xffffffff, 0x03860204),
196 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCR_GENERAL_CNTL, 0x1ff0ffff, 0x00000500),
197 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGE_PRIV_CONTROL, 0x00007fff, 0x000001fe),
198 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL1_PIPE_STEER, 0xffffffff, 0xe4e4e4e4),
199 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2_PIPE_STEER_0, 0x77777777, 0x10321032),
200 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2_PIPE_STEER_1, 0x77777777, 0x02310231),
201 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2A_ADDR_MATCH_MASK, 0xffffffff, 0xffffffcf),
202 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_ADDR_MATCH_MASK, 0xffffffff, 0xffffffcf),
203 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CGTT_SCLK_CTRL, 0xffff0fff, 0x10000100),
204 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CTRL2, 0xffffffff, 0x1402002f),
205 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGL2C_CTRL3, 0xffffbfff, 0x00000188),
206 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_0, 0xffffffff, 0x842a4c02),
207 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_TIMEOUT_COUNTER, 0xffffffff, 0x00000800),
208 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x08000009),
209 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04440000),
210 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_2, 0x00000820, 0x00000820),
211 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
212 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_SPARE, 0xffffffff, 0xffff3101),
213 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_ALU_CLK_CTRL, 0xffffffff, 0xffffffff),
214 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_ARB_CONFIG, 0x00000133, 0x00000130),
215 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_LDS_CLK_CTRL, 0xffffffff, 0xffffffff),
216 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfff7ffff, 0x01030000),
217 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CNTL, 0xffdf80ff, 0x479c0010),
218 SOC15_REG_GOLDEN_VALUE(GC, 0, mmUTCL1_CTRL, 0xffffffff, 0x00800000)
219 };
220
221 static const struct soc15_reg_golden golden_settings_gc_10_1_nv14[] =
222 {
223
224 };
225
226 static const struct soc15_reg_golden golden_settings_gc_10_1_2_nv12[] =
227 {
228
229 };
230
231 #define DEFAULT_SH_MEM_CONFIG \
232 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \
233 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \
234 (SH_MEM_RETRY_MODE_ALL << SH_MEM_CONFIG__RETRY_MODE__SHIFT) | \
235 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT))
236
237
238 static void gfx_v10_0_set_ring_funcs(struct amdgpu_device *adev);
239 static void gfx_v10_0_set_irq_funcs(struct amdgpu_device *adev);
240 static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev);
241 static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev);
242 static int gfx_v10_0_get_cu_info(struct amdgpu_device *adev,
243 struct amdgpu_cu_info *cu_info);
244 static uint64_t gfx_v10_0_get_gpu_clock_counter(struct amdgpu_device *adev);
245 static void gfx_v10_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
246 u32 sh_num, u32 instance);
247 static u32 gfx_v10_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev);
248
249 static int gfx_v10_0_rlc_backdoor_autoload_buffer_init(struct amdgpu_device *adev);
250 static void gfx_v10_0_rlc_backdoor_autoload_buffer_fini(struct amdgpu_device *adev);
251 static int gfx_v10_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev);
252 static int gfx_v10_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev);
253 static void gfx_v10_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume);
254 static void gfx_v10_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume);
255 static void gfx_v10_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start);
256
257 static void gfx10_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask)
258 {
259 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
260 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) |
261 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
262 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask));
263 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask));
264 amdgpu_ring_write(kiq_ring, 0);
265 amdgpu_ring_write(kiq_ring, 0);
266 amdgpu_ring_write(kiq_ring, 0);
267 amdgpu_ring_write(kiq_ring, 0);
268 }
269
270 static void gfx10_kiq_map_queues(struct amdgpu_ring *kiq_ring,
271 struct amdgpu_ring *ring)
272 {
273 struct amdgpu_device *adev = kiq_ring->adev;
274 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
275 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
276 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
277
278 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
279
280 amdgpu_ring_write(kiq_ring,
281 PACKET3_MAP_QUEUES_QUEUE_SEL(0) |
282 PACKET3_MAP_QUEUES_VMID(0) |
283 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
284 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
285 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
286 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
287 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
288 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
289 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
290 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
291 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
292 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
293 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
294 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
295 }
296
297 static void gfx10_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
298 struct amdgpu_ring *ring,
299 enum amdgpu_unmap_queues_action action,
300 u64 gpu_addr, u64 seq)
301 {
302 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
303
304 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
305 amdgpu_ring_write(kiq_ring,
306 PACKET3_UNMAP_QUEUES_ACTION(action) |
307 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
308 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
309 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
310 amdgpu_ring_write(kiq_ring,
311 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
312
313 if (action == PREEMPT_QUEUES_NO_UNMAP) {
314 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
315 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
316 amdgpu_ring_write(kiq_ring, seq);
317 } else {
318 amdgpu_ring_write(kiq_ring, 0);
319 amdgpu_ring_write(kiq_ring, 0);
320 amdgpu_ring_write(kiq_ring, 0);
321 }
322 }
323
324 static void gfx10_kiq_query_status(struct amdgpu_ring *kiq_ring,
325 struct amdgpu_ring *ring,
326 u64 addr,
327 u64 seq)
328 {
329 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
330
331 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
332 amdgpu_ring_write(kiq_ring,
333 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
334 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
335 PACKET3_QUERY_STATUS_COMMAND(2));
336 amdgpu_ring_write(kiq_ring,
337 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
338 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
339 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
340 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
341 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
342 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
343 }
344
345 static const struct kiq_pm4_funcs gfx_v10_0_kiq_pm4_funcs = {
346 .kiq_set_resources = gfx10_kiq_set_resources,
347 .kiq_map_queues = gfx10_kiq_map_queues,
348 .kiq_unmap_queues = gfx10_kiq_unmap_queues,
349 .kiq_query_status = gfx10_kiq_query_status,
350 .set_resources_size = 8,
351 .map_queues_size = 7,
352 .unmap_queues_size = 6,
353 .query_status_size = 7,
354 };
355
356 static void gfx_v10_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
357 {
358 adev->gfx.kiq.pmf = &gfx_v10_0_kiq_pm4_funcs;
359 }
360
361 static void gfx_v10_0_init_golden_registers(struct amdgpu_device *adev)
362 {
363 switch (adev->asic_type) {
364 case CHIP_NAVI10:
365 soc15_program_register_sequence(adev,
366 golden_settings_gc_10_1,
367 (const u32)ARRAY_SIZE(golden_settings_gc_10_1));
368 soc15_program_register_sequence(adev,
369 golden_settings_gc_10_0_nv10,
370 (const u32)ARRAY_SIZE(golden_settings_gc_10_0_nv10));
371 break;
372 case CHIP_NAVI14:
373 soc15_program_register_sequence(adev,
374 golden_settings_gc_10_1_1,
375 (const u32)ARRAY_SIZE(golden_settings_gc_10_1_1));
376 soc15_program_register_sequence(adev,
377 golden_settings_gc_10_1_nv14,
378 (const u32)ARRAY_SIZE(golden_settings_gc_10_1_nv14));
379 break;
380 case CHIP_NAVI12:
381 soc15_program_register_sequence(adev,
382 golden_settings_gc_10_1_2,
383 (const u32)ARRAY_SIZE(golden_settings_gc_10_1_2));
384 soc15_program_register_sequence(adev,
385 golden_settings_gc_10_1_2_nv12,
386 (const u32)ARRAY_SIZE(golden_settings_gc_10_1_2_nv12));
387 break;
388 default:
389 break;
390 }
391 }
392
393 static void gfx_v10_0_scratch_init(struct amdgpu_device *adev)
394 {
395 adev->gfx.scratch.num_reg = 8;
396 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
397 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
398 }
399
400 static void gfx_v10_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
401 bool wc, uint32_t reg, uint32_t val)
402 {
403 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
404 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
405 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0));
406 amdgpu_ring_write(ring, reg);
407 amdgpu_ring_write(ring, 0);
408 amdgpu_ring_write(ring, val);
409 }
410
411 static void gfx_v10_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
412 int mem_space, int opt, uint32_t addr0,
413 uint32_t addr1, uint32_t ref, uint32_t mask,
414 uint32_t inv)
415 {
416 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
417 amdgpu_ring_write(ring,
418
419 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
420 WAIT_REG_MEM_OPERATION(opt) |
421 WAIT_REG_MEM_FUNCTION(3) |
422 WAIT_REG_MEM_ENGINE(eng_sel)));
423
424 if (mem_space)
425 BUG_ON(addr0 & 0x3);
426 amdgpu_ring_write(ring, addr0);
427 amdgpu_ring_write(ring, addr1);
428 amdgpu_ring_write(ring, ref);
429 amdgpu_ring_write(ring, mask);
430 amdgpu_ring_write(ring, inv);
431 }
432
433 static int gfx_v10_0_ring_test_ring(struct amdgpu_ring *ring)
434 {
435 struct amdgpu_device *adev = ring->adev;
436 uint32_t scratch;
437 uint32_t tmp = 0;
438 unsigned i;
439 int r;
440
441 r = amdgpu_gfx_scratch_get(adev, &scratch);
442 if (r) {
443 DRM_ERROR("amdgpu: cp failed to get scratch reg (%d).\n", r);
444 return r;
445 }
446
447 WREG32(scratch, 0xCAFEDEAD);
448
449 r = amdgpu_ring_alloc(ring, 3);
450 if (r) {
451 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
452 ring->idx, r);
453 amdgpu_gfx_scratch_free(adev, scratch);
454 return r;
455 }
456
457 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
458 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
459 amdgpu_ring_write(ring, 0xDEADBEEF);
460 amdgpu_ring_commit(ring);
461
462 for (i = 0; i < adev->usec_timeout; i++) {
463 tmp = RREG32(scratch);
464 if (tmp == 0xDEADBEEF)
465 break;
466 if (amdgpu_emu_mode == 1)
467 msleep(1);
468 else
469 udelay(1);
470 }
471 if (i < adev->usec_timeout) {
472 if (amdgpu_emu_mode == 1)
473 DRM_INFO("ring test on %d succeeded in %d msecs\n",
474 ring->idx, i);
475 else
476 DRM_INFO("ring test on %d succeeded in %d usecs\n",
477 ring->idx, i);
478 } else {
479 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
480 ring->idx, scratch, tmp);
481 r = -EINVAL;
482 }
483 amdgpu_gfx_scratch_free(adev, scratch);
484
485 return r;
486 }
487
488 static int gfx_v10_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
489 {
490 struct amdgpu_device *adev = ring->adev;
491 struct amdgpu_ib ib;
492 struct dma_fence *f = NULL;
493 uint32_t scratch;
494 uint32_t tmp = 0;
495 long r;
496
497 r = amdgpu_gfx_scratch_get(adev, &scratch);
498 if (r) {
499 DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
500 return r;
501 }
502
503 WREG32(scratch, 0xCAFEDEAD);
504
505 memset(&ib, 0, sizeof(ib));
506 r = amdgpu_ib_get(adev, NULL, 256, &ib);
507 if (r) {
508 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
509 goto err1;
510 }
511
512 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
513 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
514 ib.ptr[2] = 0xDEADBEEF;
515 ib.length_dw = 3;
516
517 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
518 if (r)
519 goto err2;
520
521 r = dma_fence_wait_timeout(f, false, timeout);
522 if (r == 0) {
523 DRM_ERROR("amdgpu: IB test timed out.\n");
524 r = -ETIMEDOUT;
525 goto err2;
526 } else if (r < 0) {
527 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
528 goto err2;
529 }
530
531 tmp = RREG32(scratch);
532 if (tmp == 0xDEADBEEF) {
533 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
534 r = 0;
535 } else {
536 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
537 scratch, tmp);
538 r = -EINVAL;
539 }
540 err2:
541 amdgpu_ib_free(adev, &ib, NULL);
542 dma_fence_put(f);
543 err1:
544 amdgpu_gfx_scratch_free(adev, scratch);
545
546 return r;
547 }
548
549 static void gfx_v10_0_free_microcode(struct amdgpu_device *adev)
550 {
551 release_firmware(adev->gfx.pfp_fw);
552 adev->gfx.pfp_fw = NULL;
553 release_firmware(adev->gfx.me_fw);
554 adev->gfx.me_fw = NULL;
555 release_firmware(adev->gfx.ce_fw);
556 adev->gfx.ce_fw = NULL;
557 release_firmware(adev->gfx.rlc_fw);
558 adev->gfx.rlc_fw = NULL;
559 release_firmware(adev->gfx.mec_fw);
560 adev->gfx.mec_fw = NULL;
561 release_firmware(adev->gfx.mec2_fw);
562 adev->gfx.mec2_fw = NULL;
563
564 kfree(adev->gfx.rlc.register_list_format);
565 }
566
567 static void gfx_v10_0_check_fw_write_wait(struct amdgpu_device *adev)
568 {
569 adev->gfx.cp_fw_write_wait = false;
570
571 switch (adev->asic_type) {
572 case CHIP_NAVI10:
573 case CHIP_NAVI12:
574 case CHIP_NAVI14:
575 if ((adev->gfx.me_fw_version >= 0x00000046) &&
576 (adev->gfx.me_feature_version >= 27) &&
577 (adev->gfx.pfp_fw_version >= 0x00000068) &&
578 (adev->gfx.pfp_feature_version >= 27) &&
579 (adev->gfx.mec_fw_version >= 0x0000005b) &&
580 (adev->gfx.mec_feature_version >= 27))
581 adev->gfx.cp_fw_write_wait = true;
582 break;
583 default:
584 break;
585 }
586
587 if (adev->gfx.cp_fw_write_wait == false)
588 DRM_WARN_ONCE("Warning: check cp_fw_version and update it to realize \
589 GRBM requires 1-cycle delay in cp firmware\n");
590 }
591
592
593 static void gfx_v10_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
594 {
595 const struct rlc_firmware_header_v2_1 *rlc_hdr;
596
597 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
598 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
599 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
600 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
601 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
602 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
603 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
604 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
605 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
606 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
607 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
608 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
609 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
610 adev->gfx.rlc.reg_list_format_direct_reg_list_length =
611 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
612 }
613
614 static void gfx_v10_0_check_gfxoff_flag(struct amdgpu_device *adev)
615 {
616 switch (adev->asic_type) {
617 case CHIP_NAVI10:
618 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
619 break;
620 default:
621 break;
622 }
623 }
624
625 static int gfx_v10_0_init_microcode(struct amdgpu_device *adev)
626 {
627 const char *chip_name;
628 char fw_name[40];
629 char wks[10];
630 int err;
631 struct amdgpu_firmware_info *info = NULL;
632 const struct common_firmware_header *header = NULL;
633 const struct gfx_firmware_header_v1_0 *cp_hdr;
634 const struct rlc_firmware_header_v2_0 *rlc_hdr;
635 unsigned int *tmp = NULL;
636 unsigned int i = 0;
637 uint16_t version_major;
638 uint16_t version_minor;
639
640 DRM_DEBUG("\n");
641
642 memset(wks, 0, sizeof(wks));
643 switch (adev->asic_type) {
644 case CHIP_NAVI10:
645 chip_name = "navi10";
646 break;
647 case CHIP_NAVI14:
648 chip_name = "navi14";
649 if (!(adev->pdev->device == 0x7340 &&
650 adev->pdev->revision != 0x00))
651 snprintf(wks, sizeof(wks), "_wks");
652 break;
653 case CHIP_NAVI12:
654 chip_name = "navi12";
655 break;
656 default:
657 BUG();
658 }
659
660 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp%s.bin", chip_name, wks);
661 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
662 if (err)
663 goto out;
664 err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
665 if (err)
666 goto out;
667 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
668 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
669 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
670
671 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me%s.bin", chip_name, wks);
672 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
673 if (err)
674 goto out;
675 err = amdgpu_ucode_validate(adev->gfx.me_fw);
676 if (err)
677 goto out;
678 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
679 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
680 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
681
682 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce%s.bin", chip_name, wks);
683 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
684 if (err)
685 goto out;
686 err = amdgpu_ucode_validate(adev->gfx.ce_fw);
687 if (err)
688 goto out;
689 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
690 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
691 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
692
693 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
694 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
695 if (err)
696 goto out;
697 err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
698 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
699 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
700 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
701 if (version_major == 2 && version_minor == 1)
702 adev->gfx.rlc.is_rlc_v2_1 = true;
703
704 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
705 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
706 adev->gfx.rlc.save_and_restore_offset =
707 le32_to_cpu(rlc_hdr->save_and_restore_offset);
708 adev->gfx.rlc.clear_state_descriptor_offset =
709 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
710 adev->gfx.rlc.avail_scratch_ram_locations =
711 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
712 adev->gfx.rlc.reg_restore_list_size =
713 le32_to_cpu(rlc_hdr->reg_restore_list_size);
714 adev->gfx.rlc.reg_list_format_start =
715 le32_to_cpu(rlc_hdr->reg_list_format_start);
716 adev->gfx.rlc.reg_list_format_separate_start =
717 le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
718 adev->gfx.rlc.starting_offsets_start =
719 le32_to_cpu(rlc_hdr->starting_offsets_start);
720 adev->gfx.rlc.reg_list_format_size_bytes =
721 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
722 adev->gfx.rlc.reg_list_size_bytes =
723 le32_to_cpu(rlc_hdr->reg_list_size_bytes);
724 adev->gfx.rlc.register_list_format =
725 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
726 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
727 if (!adev->gfx.rlc.register_list_format) {
728 err = -ENOMEM;
729 goto out;
730 }
731
732 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
733 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
734 for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
735 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
736
737 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
738
739 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
740 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
741 for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
742 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
743
744 if (adev->gfx.rlc.is_rlc_v2_1)
745 gfx_v10_0_init_rlc_ext_microcode(adev);
746
747 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec%s.bin", chip_name, wks);
748 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
749 if (err)
750 goto out;
751 err = amdgpu_ucode_validate(adev->gfx.mec_fw);
752 if (err)
753 goto out;
754 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
755 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
756 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
757
758 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2%s.bin", chip_name, wks);
759 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
760 if (!err) {
761 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
762 if (err)
763 goto out;
764 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
765 adev->gfx.mec2_fw->data;
766 adev->gfx.mec2_fw_version =
767 le32_to_cpu(cp_hdr->header.ucode_version);
768 adev->gfx.mec2_feature_version =
769 le32_to_cpu(cp_hdr->ucode_feature_version);
770 } else {
771 err = 0;
772 adev->gfx.mec2_fw = NULL;
773 }
774
775 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
776 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
777 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
778 info->fw = adev->gfx.pfp_fw;
779 header = (const struct common_firmware_header *)info->fw->data;
780 adev->firmware.fw_size +=
781 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
782
783 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
784 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
785 info->fw = adev->gfx.me_fw;
786 header = (const struct common_firmware_header *)info->fw->data;
787 adev->firmware.fw_size +=
788 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
789
790 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
791 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
792 info->fw = adev->gfx.ce_fw;
793 header = (const struct common_firmware_header *)info->fw->data;
794 adev->firmware.fw_size +=
795 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
796
797 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
798 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
799 info->fw = adev->gfx.rlc_fw;
800 header = (const struct common_firmware_header *)info->fw->data;
801 adev->firmware.fw_size +=
802 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
803
804 if (adev->gfx.rlc.is_rlc_v2_1 &&
805 adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
806 adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
807 adev->gfx.rlc.save_restore_list_srm_size_bytes) {
808 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
809 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
810 info->fw = adev->gfx.rlc_fw;
811 adev->firmware.fw_size +=
812 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
813
814 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
815 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
816 info->fw = adev->gfx.rlc_fw;
817 adev->firmware.fw_size +=
818 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
819
820 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
821 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
822 info->fw = adev->gfx.rlc_fw;
823 adev->firmware.fw_size +=
824 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
825 }
826
827 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
828 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
829 info->fw = adev->gfx.mec_fw;
830 header = (const struct common_firmware_header *)info->fw->data;
831 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
832 adev->firmware.fw_size +=
833 ALIGN(le32_to_cpu(header->ucode_size_bytes) -
834 le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
835
836 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
837 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
838 info->fw = adev->gfx.mec_fw;
839 adev->firmware.fw_size +=
840 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
841
842 if (adev->gfx.mec2_fw) {
843 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
844 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
845 info->fw = adev->gfx.mec2_fw;
846 header = (const struct common_firmware_header *)info->fw->data;
847 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
848 adev->firmware.fw_size +=
849 ALIGN(le32_to_cpu(header->ucode_size_bytes) -
850 le32_to_cpu(cp_hdr->jt_size) * 4,
851 PAGE_SIZE);
852 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
853 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
854 info->fw = adev->gfx.mec2_fw;
855 adev->firmware.fw_size +=
856 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
857 PAGE_SIZE);
858 }
859 }
860
861 gfx_v10_0_check_fw_write_wait(adev);
862 out:
863 if (err) {
864 dev_err(adev->dev,
865 "gfx10: Failed to load firmware \"%s\"\n",
866 fw_name);
867 release_firmware(adev->gfx.pfp_fw);
868 adev->gfx.pfp_fw = NULL;
869 release_firmware(adev->gfx.me_fw);
870 adev->gfx.me_fw = NULL;
871 release_firmware(adev->gfx.ce_fw);
872 adev->gfx.ce_fw = NULL;
873 release_firmware(adev->gfx.rlc_fw);
874 adev->gfx.rlc_fw = NULL;
875 release_firmware(adev->gfx.mec_fw);
876 adev->gfx.mec_fw = NULL;
877 release_firmware(adev->gfx.mec2_fw);
878 adev->gfx.mec2_fw = NULL;
879 }
880
881 gfx_v10_0_check_gfxoff_flag(adev);
882
883 return err;
884 }
885
886 static u32 gfx_v10_0_get_csb_size(struct amdgpu_device *adev)
887 {
888 u32 count = 0;
889 const struct cs_section_def *sect = NULL;
890 const struct cs_extent_def *ext = NULL;
891
892
893 count += 2;
894
895 count += 3;
896
897 for (sect = gfx10_cs_data; sect->section != NULL; ++sect) {
898 for (ext = sect->section; ext->extent != NULL; ++ext) {
899 if (sect->id == SECT_CONTEXT)
900 count += 2 + ext->reg_count;
901 else
902 return 0;
903 }
904 }
905
906
907 count += 3;
908
909 count += 2;
910
911 count += 2;
912
913 return count;
914 }
915
916 static void gfx_v10_0_get_csb_buffer(struct amdgpu_device *adev,
917 volatile u32 *buffer)
918 {
919 u32 count = 0, i;
920 const struct cs_section_def *sect = NULL;
921 const struct cs_extent_def *ext = NULL;
922 int ctx_reg_offset;
923
924 if (adev->gfx.rlc.cs_data == NULL)
925 return;
926 if (buffer == NULL)
927 return;
928
929 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
930 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
931
932 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
933 buffer[count++] = cpu_to_le32(0x80000000);
934 buffer[count++] = cpu_to_le32(0x80000000);
935
936 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
937 for (ext = sect->section; ext->extent != NULL; ++ext) {
938 if (sect->id == SECT_CONTEXT) {
939 buffer[count++] =
940 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
941 buffer[count++] = cpu_to_le32(ext->reg_index -
942 PACKET3_SET_CONTEXT_REG_START);
943 for (i = 0; i < ext->reg_count; i++)
944 buffer[count++] = cpu_to_le32(ext->extent[i]);
945 } else {
946 return;
947 }
948 }
949 }
950
951 ctx_reg_offset =
952 SOC15_REG_OFFSET(GC, 0, mmPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
953 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
954 buffer[count++] = cpu_to_le32(ctx_reg_offset);
955 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override);
956
957 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
958 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
959
960 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
961 buffer[count++] = cpu_to_le32(0);
962 }
963
964 static void gfx_v10_0_rlc_fini(struct amdgpu_device *adev)
965 {
966
967 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
968 &adev->gfx.rlc.clear_state_gpu_addr,
969 (void **)&adev->gfx.rlc.cs_ptr);
970
971
972 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
973 &adev->gfx.rlc.cp_table_gpu_addr,
974 (void **)&adev->gfx.rlc.cp_table_ptr);
975 }
976
977 static int gfx_v10_0_rlc_init(struct amdgpu_device *adev)
978 {
979 const struct cs_section_def *cs_data;
980 int r;
981
982 adev->gfx.rlc.cs_data = gfx10_cs_data;
983
984 cs_data = adev->gfx.rlc.cs_data;
985
986 if (cs_data) {
987
988 r = amdgpu_gfx_rlc_init_csb(adev);
989 if (r)
990 return r;
991 }
992
993 return 0;
994 }
995
996 static int gfx_v10_0_csb_vram_pin(struct amdgpu_device *adev)
997 {
998 int r;
999
1000 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
1001 if (unlikely(r != 0))
1002 return r;
1003
1004 r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj,
1005 AMDGPU_GEM_DOMAIN_VRAM);
1006 if (!r)
1007 adev->gfx.rlc.clear_state_gpu_addr =
1008 amdgpu_bo_gpu_offset(adev->gfx.rlc.clear_state_obj);
1009
1010 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
1011
1012 return r;
1013 }
1014
1015 static void gfx_v10_0_csb_vram_unpin(struct amdgpu_device *adev)
1016 {
1017 int r;
1018
1019 if (!adev->gfx.rlc.clear_state_obj)
1020 return;
1021
1022 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
1023 if (likely(r == 0)) {
1024 amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
1025 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
1026 }
1027 }
1028
1029 static void gfx_v10_0_mec_fini(struct amdgpu_device *adev)
1030 {
1031 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1032 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1033 }
1034
1035 static int gfx_v10_0_me_init(struct amdgpu_device *adev)
1036 {
1037 int r;
1038
1039 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES);
1040
1041 amdgpu_gfx_graphics_queue_acquire(adev);
1042
1043 r = gfx_v10_0_init_microcode(adev);
1044 if (r)
1045 DRM_ERROR("Failed to load gfx firmware!\n");
1046
1047 return r;
1048 }
1049
1050 static int gfx_v10_0_mec_init(struct amdgpu_device *adev)
1051 {
1052 int r;
1053 u32 *hpd;
1054 const __le32 *fw_data = NULL;
1055 unsigned fw_size;
1056 u32 *fw = NULL;
1057 size_t mec_hpd_size;
1058
1059 const struct gfx_firmware_header_v1_0 *mec_hdr = NULL;
1060
1061 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1062
1063
1064 amdgpu_gfx_compute_queue_acquire(adev);
1065 mec_hpd_size = adev->gfx.num_compute_rings * GFX10_MEC_HPD_SIZE;
1066
1067 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1068 AMDGPU_GEM_DOMAIN_GTT,
1069 &adev->gfx.mec.hpd_eop_obj,
1070 &adev->gfx.mec.hpd_eop_gpu_addr,
1071 (void **)&hpd);
1072 if (r) {
1073 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1074 gfx_v10_0_mec_fini(adev);
1075 return r;
1076 }
1077
1078 memset(hpd, 0, adev->gfx.mec.hpd_eop_obj->tbo.mem.size);
1079
1080 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1081 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1082
1083 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1084 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1085
1086 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
1087 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1088 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes);
1089
1090 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1091 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1092 &adev->gfx.mec.mec_fw_obj,
1093 &adev->gfx.mec.mec_fw_gpu_addr,
1094 (void **)&fw);
1095 if (r) {
1096 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r);
1097 gfx_v10_0_mec_fini(adev);
1098 return r;
1099 }
1100
1101 memcpy(fw, fw_data, fw_size);
1102
1103 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1104 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1105 }
1106
1107 return 0;
1108 }
1109
1110 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address)
1111 {
1112 WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
1113 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1114 (address << SQ_IND_INDEX__INDEX__SHIFT));
1115 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1116 }
1117
1118 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave,
1119 uint32_t thread, uint32_t regno,
1120 uint32_t num, uint32_t *out)
1121 {
1122 WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
1123 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1124 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1125 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) |
1126 (SQ_IND_INDEX__AUTO_INCR_MASK));
1127 while (num--)
1128 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1129 }
1130
1131 static void gfx_v10_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1132 {
1133
1134
1135
1136 WARN_ON(simd != 0);
1137
1138
1139 dst[(*no_fields)++] = 2;
1140 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS);
1141 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO);
1142 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI);
1143 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO);
1144 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI);
1145 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1);
1146 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2);
1147 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_INST_DW0);
1148 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC);
1149 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC);
1150 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS);
1151 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS);
1152 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2);
1153 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1);
1154 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0);
1155 }
1156
1157 static void gfx_v10_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
1158 uint32_t wave, uint32_t start,
1159 uint32_t size, uint32_t *dst)
1160 {
1161 WARN_ON(simd != 0);
1162
1163 wave_read_regs(
1164 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size,
1165 dst);
1166 }
1167
1168 static void gfx_v10_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
1169 uint32_t wave, uint32_t thread,
1170 uint32_t start, uint32_t size,
1171 uint32_t *dst)
1172 {
1173 wave_read_regs(
1174 adev, wave, thread,
1175 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1176 }
1177
1178 static void gfx_v10_0_select_me_pipe_q(struct amdgpu_device *adev,
1179 u32 me, u32 pipe, u32 q, u32 vm)
1180 {
1181 nv_grbm_select(adev, me, pipe, q, vm);
1182 }
1183
1184
1185 static const struct amdgpu_gfx_funcs gfx_v10_0_gfx_funcs = {
1186 .get_gpu_clock_counter = &gfx_v10_0_get_gpu_clock_counter,
1187 .select_se_sh = &gfx_v10_0_select_se_sh,
1188 .read_wave_data = &gfx_v10_0_read_wave_data,
1189 .read_wave_sgprs = &gfx_v10_0_read_wave_sgprs,
1190 .read_wave_vgprs = &gfx_v10_0_read_wave_vgprs,
1191 .select_me_pipe_q = &gfx_v10_0_select_me_pipe_q,
1192 };
1193
1194 static void gfx_v10_0_gpu_early_init(struct amdgpu_device *adev)
1195 {
1196 u32 gb_addr_config;
1197
1198 adev->gfx.funcs = &gfx_v10_0_gfx_funcs;
1199
1200 switch (adev->asic_type) {
1201 case CHIP_NAVI10:
1202 case CHIP_NAVI14:
1203 case CHIP_NAVI12:
1204 adev->gfx.config.max_hw_contexts = 8;
1205 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1206 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1207 adev->gfx.config.sc_hiz_tile_fifo_size = 0;
1208 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1209 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1210 break;
1211 default:
1212 BUG();
1213 break;
1214 }
1215
1216 adev->gfx.config.gb_addr_config = gb_addr_config;
1217
1218 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
1219 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
1220 GB_ADDR_CONFIG, NUM_PIPES);
1221
1222 adev->gfx.config.max_tile_pipes =
1223 adev->gfx.config.gb_addr_config_fields.num_pipes;
1224
1225 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
1226 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
1227 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS);
1228 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
1229 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
1230 GB_ADDR_CONFIG, NUM_RB_PER_SE);
1231 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
1232 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
1233 GB_ADDR_CONFIG, NUM_SHADER_ENGINES);
1234 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
1235 REG_GET_FIELD(adev->gfx.config.gb_addr_config,
1236 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE));
1237 }
1238
1239 static int gfx_v10_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id,
1240 int me, int pipe, int queue)
1241 {
1242 int r;
1243 struct amdgpu_ring *ring;
1244 unsigned int irq_type;
1245
1246 ring = &adev->gfx.gfx_ring[ring_id];
1247
1248 ring->me = me;
1249 ring->pipe = pipe;
1250 ring->queue = queue;
1251
1252 ring->ring_obj = NULL;
1253 ring->use_doorbell = true;
1254
1255 if (!ring_id)
1256 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
1257 else
1258 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1;
1259 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1260
1261 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe;
1262 r = amdgpu_ring_init(adev, ring, 1024,
1263 &adev->gfx.eop_irq, irq_type);
1264 if (r)
1265 return r;
1266 return 0;
1267 }
1268
1269 static int gfx_v10_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1270 int mec, int pipe, int queue)
1271 {
1272 int r;
1273 unsigned irq_type;
1274 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1275
1276 ring = &adev->gfx.compute_ring[ring_id];
1277
1278
1279 ring->me = mec + 1;
1280 ring->pipe = pipe;
1281 ring->queue = queue;
1282
1283 ring->ring_obj = NULL;
1284 ring->use_doorbell = true;
1285 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
1286 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1287 + (ring_id * GFX10_MEC_HPD_SIZE);
1288 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1289
1290 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1291 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1292 + ring->pipe;
1293
1294
1295 r = amdgpu_ring_init(adev, ring, 1024,
1296 &adev->gfx.eop_irq, irq_type);
1297 if (r)
1298 return r;
1299
1300 return 0;
1301 }
1302
1303 static int gfx_v10_0_sw_init(void *handle)
1304 {
1305 int i, j, k, r, ring_id = 0;
1306 struct amdgpu_kiq *kiq;
1307 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1308
1309 switch (adev->asic_type) {
1310 case CHIP_NAVI10:
1311 case CHIP_NAVI14:
1312 case CHIP_NAVI12:
1313 adev->gfx.me.num_me = 1;
1314 adev->gfx.me.num_pipe_per_me = 2;
1315 adev->gfx.me.num_queue_per_pipe = 1;
1316 adev->gfx.mec.num_mec = 2;
1317 adev->gfx.mec.num_pipe_per_mec = 4;
1318 adev->gfx.mec.num_queue_per_pipe = 8;
1319 break;
1320 default:
1321 adev->gfx.me.num_me = 1;
1322 adev->gfx.me.num_pipe_per_me = 1;
1323 adev->gfx.me.num_queue_per_pipe = 1;
1324 adev->gfx.mec.num_mec = 1;
1325 adev->gfx.mec.num_pipe_per_mec = 4;
1326 adev->gfx.mec.num_queue_per_pipe = 8;
1327 break;
1328 }
1329
1330
1331 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
1332 GFX_10_1__SRCID__CP_IB2_INTERRUPT_PKT,
1333 &adev->gfx.kiq.irq);
1334 if (r)
1335 return r;
1336
1337
1338 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP,
1339 GFX_10_1__SRCID__CP_EOP_INTERRUPT,
1340 &adev->gfx.eop_irq);
1341 if (r)
1342 return r;
1343
1344
1345 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_10_1__SRCID__CP_PRIV_REG_FAULT,
1346 &adev->gfx.priv_reg_irq);
1347 if (r)
1348 return r;
1349
1350
1351 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_10_1__SRCID__CP_PRIV_INSTR_FAULT,
1352 &adev->gfx.priv_inst_irq);
1353 if (r)
1354 return r;
1355
1356 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1357
1358 gfx_v10_0_scratch_init(adev);
1359
1360 r = gfx_v10_0_me_init(adev);
1361 if (r)
1362 return r;
1363
1364 r = gfx_v10_0_rlc_init(adev);
1365 if (r) {
1366 DRM_ERROR("Failed to init rlc BOs!\n");
1367 return r;
1368 }
1369
1370 r = gfx_v10_0_mec_init(adev);
1371 if (r) {
1372 DRM_ERROR("Failed to init MEC BOs!\n");
1373 return r;
1374 }
1375
1376
1377 for (i = 0; i < adev->gfx.me.num_me; i++) {
1378 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) {
1379 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) {
1380 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j))
1381 continue;
1382
1383 r = gfx_v10_0_gfx_ring_init(adev, ring_id,
1384 i, k, j);
1385 if (r)
1386 return r;
1387 ring_id++;
1388 }
1389 }
1390 }
1391
1392 ring_id = 0;
1393
1394 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1395 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1396 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1397 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k,
1398 j))
1399 continue;
1400
1401 r = gfx_v10_0_compute_ring_init(adev, ring_id,
1402 i, k, j);
1403 if (r)
1404 return r;
1405
1406 ring_id++;
1407 }
1408 }
1409 }
1410
1411 r = amdgpu_gfx_kiq_init(adev, GFX10_MEC_HPD_SIZE);
1412 if (r) {
1413 DRM_ERROR("Failed to init KIQ BOs!\n");
1414 return r;
1415 }
1416
1417 kiq = &adev->gfx.kiq;
1418 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
1419 if (r)
1420 return r;
1421
1422 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v10_compute_mqd));
1423 if (r)
1424 return r;
1425
1426
1427 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1428 r = gfx_v10_0_rlc_backdoor_autoload_buffer_init(adev);
1429 if (r)
1430 return r;
1431 }
1432
1433 adev->gfx.ce_ram_size = F32_CE_PROGRAM_RAM_SIZE;
1434
1435 gfx_v10_0_gpu_early_init(adev);
1436
1437 return 0;
1438 }
1439
1440 static void gfx_v10_0_pfp_fini(struct amdgpu_device *adev)
1441 {
1442 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj,
1443 &adev->gfx.pfp.pfp_fw_gpu_addr,
1444 (void **)&adev->gfx.pfp.pfp_fw_ptr);
1445 }
1446
1447 static void gfx_v10_0_ce_fini(struct amdgpu_device *adev)
1448 {
1449 amdgpu_bo_free_kernel(&adev->gfx.ce.ce_fw_obj,
1450 &adev->gfx.ce.ce_fw_gpu_addr,
1451 (void **)&adev->gfx.ce.ce_fw_ptr);
1452 }
1453
1454 static void gfx_v10_0_me_fini(struct amdgpu_device *adev)
1455 {
1456 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj,
1457 &adev->gfx.me.me_fw_gpu_addr,
1458 (void **)&adev->gfx.me.me_fw_ptr);
1459 }
1460
1461 static int gfx_v10_0_sw_fini(void *handle)
1462 {
1463 int i;
1464 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1465
1466 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
1467 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
1468 for (i = 0; i < adev->gfx.num_compute_rings; i++)
1469 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
1470
1471 amdgpu_gfx_mqd_sw_fini(adev);
1472 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
1473 amdgpu_gfx_kiq_fini(adev);
1474
1475 gfx_v10_0_pfp_fini(adev);
1476 gfx_v10_0_ce_fini(adev);
1477 gfx_v10_0_me_fini(adev);
1478 gfx_v10_0_rlc_fini(adev);
1479 gfx_v10_0_mec_fini(adev);
1480
1481 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO)
1482 gfx_v10_0_rlc_backdoor_autoload_buffer_fini(adev);
1483
1484 gfx_v10_0_free_microcode(adev);
1485
1486 return 0;
1487 }
1488
1489
1490 static void gfx_v10_0_tiling_mode_table_init(struct amdgpu_device *adev)
1491 {
1492
1493 }
1494
1495 static void gfx_v10_0_select_se_sh(struct amdgpu_device *adev, u32 se_num,
1496 u32 sh_num, u32 instance)
1497 {
1498 u32 data;
1499
1500 if (instance == 0xffffffff)
1501 data = REG_SET_FIELD(0, GRBM_GFX_INDEX,
1502 INSTANCE_BROADCAST_WRITES, 1);
1503 else
1504 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX,
1505 instance);
1506
1507 if (se_num == 0xffffffff)
1508 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES,
1509 1);
1510 else
1511 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
1512
1513 if (sh_num == 0xffffffff)
1514 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES,
1515 1);
1516 else
1517 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num);
1518
1519 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, data);
1520 }
1521
1522 static u32 gfx_v10_0_get_rb_active_bitmap(struct amdgpu_device *adev)
1523 {
1524 u32 data, mask;
1525
1526 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
1527 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
1528
1529 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
1530 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
1531
1532 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
1533 adev->gfx.config.max_sh_per_se);
1534
1535 return (~data) & mask;
1536 }
1537
1538 static void gfx_v10_0_setup_rb(struct amdgpu_device *adev)
1539 {
1540 int i, j;
1541 u32 data;
1542 u32 active_rbs = 0;
1543 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
1544 adev->gfx.config.max_sh_per_se;
1545
1546 mutex_lock(&adev->grbm_idx_mutex);
1547 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1548 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1549 gfx_v10_0_select_se_sh(adev, i, j, 0xffffffff);
1550 data = gfx_v10_0_get_rb_active_bitmap(adev);
1551 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
1552 rb_bitmap_width_per_sh);
1553 }
1554 }
1555 gfx_v10_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1556 mutex_unlock(&adev->grbm_idx_mutex);
1557
1558 adev->gfx.config.backend_enable_mask = active_rbs;
1559 adev->gfx.config.num_rbs = hweight32(active_rbs);
1560 }
1561
1562 static u32 gfx_v10_0_init_pa_sc_tile_steering_override(struct amdgpu_device *adev)
1563 {
1564 uint32_t num_sc;
1565 uint32_t enabled_rb_per_sh;
1566 uint32_t active_rb_bitmap;
1567 uint32_t num_rb_per_sc;
1568 uint32_t num_packer_per_sc;
1569 uint32_t pa_sc_tile_steering_override;
1570
1571
1572 num_sc = adev->gfx.config.max_shader_engines * adev->gfx.config.max_sh_per_se *
1573 adev->gfx.config.num_sc_per_sh;
1574
1575 active_rb_bitmap = gfx_v10_0_get_rb_active_bitmap(adev);
1576 enabled_rb_per_sh = hweight32(active_rb_bitmap);
1577 num_rb_per_sc = enabled_rb_per_sh / adev->gfx.config.num_sc_per_sh;
1578
1579 num_packer_per_sc = adev->gfx.config.num_packer_per_sc;
1580
1581 pa_sc_tile_steering_override = 0;
1582 pa_sc_tile_steering_override |=
1583 (order_base_2(num_sc) << PA_SC_TILE_STEERING_OVERRIDE__NUM_SC__SHIFT) &
1584 PA_SC_TILE_STEERING_OVERRIDE__NUM_SC_MASK;
1585 pa_sc_tile_steering_override |=
1586 (order_base_2(num_rb_per_sc) << PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SC__SHIFT) &
1587 PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SC_MASK;
1588 pa_sc_tile_steering_override |=
1589 (order_base_2(num_packer_per_sc) << PA_SC_TILE_STEERING_OVERRIDE__NUM_PACKER_PER_SC__SHIFT) &
1590 PA_SC_TILE_STEERING_OVERRIDE__NUM_PACKER_PER_SC_MASK;
1591
1592 return pa_sc_tile_steering_override;
1593 }
1594
1595 #define DEFAULT_SH_MEM_BASES (0x6000)
1596 #define FIRST_COMPUTE_VMID (8)
1597 #define LAST_COMPUTE_VMID (16)
1598
1599 static void gfx_v10_0_init_compute_vmid(struct amdgpu_device *adev)
1600 {
1601 int i;
1602 uint32_t sh_mem_bases;
1603
1604
1605
1606
1607
1608
1609
1610 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
1611
1612 mutex_lock(&adev->srbm_mutex);
1613 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
1614 nv_grbm_select(adev, 0, 0, 0, i);
1615
1616 WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1617 WREG32_SOC15(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
1618 }
1619 nv_grbm_select(adev, 0, 0, 0, 0);
1620 mutex_unlock(&adev->srbm_mutex);
1621
1622
1623
1624 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
1625 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
1626 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
1627 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
1628 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
1629 }
1630 }
1631
1632 static void gfx_v10_0_init_gds_vmid(struct amdgpu_device *adev)
1633 {
1634 int vmid;
1635
1636
1637
1638
1639
1640
1641
1642 for (vmid = 1; vmid < 16; vmid++) {
1643 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
1644 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
1645 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
1646 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
1647 }
1648 }
1649
1650
1651 static void gfx_v10_0_tcp_harvest(struct amdgpu_device *adev)
1652 {
1653 int i, j, k;
1654 int max_wgp_per_sh = adev->gfx.config.max_cu_per_sh >> 1;
1655 u32 tmp, wgp_active_bitmap = 0;
1656 u32 gcrd_targets_disable_tcp = 0;
1657 u32 utcl_invreq_disable = 0;
1658
1659
1660
1661
1662
1663 u32 gcrd_targets_disable_mask = amdgpu_gfx_create_bitmask(
1664 2 * max_wgp_per_sh +
1665 max_wgp_per_sh +
1666 4);
1667
1668
1669
1670
1671
1672 u32 utcl_invreq_disable_mask = amdgpu_gfx_create_bitmask(
1673 2 * max_wgp_per_sh +
1674 2 * max_wgp_per_sh +
1675 4 +
1676 1);
1677
1678 if (adev->asic_type == CHIP_NAVI10 ||
1679 adev->asic_type == CHIP_NAVI14 ||
1680 adev->asic_type == CHIP_NAVI12) {
1681 mutex_lock(&adev->grbm_idx_mutex);
1682 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1683 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1684 gfx_v10_0_select_se_sh(adev, i, j, 0xffffffff);
1685 wgp_active_bitmap = gfx_v10_0_get_wgp_active_bitmap_per_sh(adev);
1686
1687
1688
1689
1690 gcrd_targets_disable_tcp = 0;
1691
1692 utcl_invreq_disable = 0;
1693
1694 for (k = 0; k < max_wgp_per_sh; k++) {
1695 if (!(wgp_active_bitmap & (1 << k))) {
1696 gcrd_targets_disable_tcp |= 3 << (2 * k);
1697 utcl_invreq_disable |= (3 << (2 * k)) |
1698 (3 << (2 * (max_wgp_per_sh + k)));
1699 }
1700 }
1701
1702 tmp = RREG32_SOC15(GC, 0, mmUTCL1_UTCL0_INVREQ_DISABLE);
1703
1704 tmp &= 0xffffffff << (4 * max_wgp_per_sh);
1705 tmp |= (utcl_invreq_disable & utcl_invreq_disable_mask);
1706 WREG32_SOC15(GC, 0, mmUTCL1_UTCL0_INVREQ_DISABLE, tmp);
1707
1708 tmp = RREG32_SOC15(GC, 0, mmGCRD_SA_TARGETS_DISABLE);
1709
1710 tmp &= 0xffffffff << (2 * max_wgp_per_sh);
1711 tmp |= (gcrd_targets_disable_tcp & gcrd_targets_disable_mask);
1712 WREG32_SOC15(GC, 0, mmGCRD_SA_TARGETS_DISABLE, tmp);
1713 }
1714 }
1715
1716 gfx_v10_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1717 mutex_unlock(&adev->grbm_idx_mutex);
1718 }
1719 }
1720
1721 static void gfx_v10_0_get_tcc_info(struct amdgpu_device *adev)
1722 {
1723
1724 uint32_t tcc_disable = RREG32_SOC15(GC, 0, mmCGTS_TCC_DISABLE) |
1725 RREG32_SOC15(GC, 0, mmCGTS_USER_TCC_DISABLE);
1726
1727 adev->gfx.config.tcc_disabled_mask =
1728 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) |
1729 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16);
1730 }
1731
1732 static void gfx_v10_0_constants_init(struct amdgpu_device *adev)
1733 {
1734 u32 tmp;
1735 int i;
1736
1737 WREG32_FIELD15(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
1738
1739 gfx_v10_0_tiling_mode_table_init(adev);
1740
1741 gfx_v10_0_setup_rb(adev);
1742 gfx_v10_0_get_cu_info(adev, &adev->gfx.cu_info);
1743 gfx_v10_0_get_tcc_info(adev);
1744 adev->gfx.config.pa_sc_tile_steering_override =
1745 gfx_v10_0_init_pa_sc_tile_steering_override(adev);
1746
1747
1748
1749 mutex_lock(&adev->srbm_mutex);
1750 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
1751 nv_grbm_select(adev, 0, 0, 0, i);
1752
1753 WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG);
1754 if (i != 0) {
1755 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
1756 (adev->gmc.private_aperture_start >> 48));
1757 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
1758 (adev->gmc.shared_aperture_start >> 48));
1759 WREG32_SOC15(GC, 0, mmSH_MEM_BASES, tmp);
1760 }
1761 }
1762 nv_grbm_select(adev, 0, 0, 0, 0);
1763
1764 mutex_unlock(&adev->srbm_mutex);
1765
1766 gfx_v10_0_init_compute_vmid(adev);
1767 gfx_v10_0_init_gds_vmid(adev);
1768
1769 }
1770
1771 static void gfx_v10_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
1772 bool enable)
1773 {
1774 u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
1775
1776 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE,
1777 enable ? 1 : 0);
1778 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE,
1779 enable ? 1 : 0);
1780 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE,
1781 enable ? 1 : 0);
1782 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE,
1783 enable ? 1 : 0);
1784
1785 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
1786 }
1787
1788 static int gfx_v10_0_init_csb(struct amdgpu_device *adev)
1789 {
1790 int r;
1791
1792 if (adev->in_gpu_reset) {
1793 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
1794 if (r)
1795 return r;
1796
1797 r = amdgpu_bo_kmap(adev->gfx.rlc.clear_state_obj,
1798 (void **)&adev->gfx.rlc.cs_ptr);
1799 if (!r) {
1800 adev->gfx.rlc.funcs->get_csb_buffer(adev,
1801 adev->gfx.rlc.cs_ptr);
1802 amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj);
1803 }
1804
1805 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
1806 if (r)
1807 return r;
1808 }
1809
1810
1811 WREG32_SOC15(GC, 0, mmRLC_CSIB_ADDR_HI,
1812 adev->gfx.rlc.clear_state_gpu_addr >> 32);
1813 WREG32_SOC15(GC, 0, mmRLC_CSIB_ADDR_LO,
1814 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
1815 WREG32_SOC15(GC, 0, mmRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size);
1816
1817 return 0;
1818 }
1819
1820 static int gfx_v10_0_init_pg(struct amdgpu_device *adev)
1821 {
1822 int i;
1823 int r;
1824
1825 r = gfx_v10_0_init_csb(adev);
1826 if (r)
1827 return r;
1828
1829 for (i = 0; i < adev->num_vmhubs; i++)
1830 amdgpu_gmc_flush_gpu_tlb(adev, 0, i, 0);
1831
1832
1833 return 0;
1834 }
1835
1836 void gfx_v10_0_rlc_stop(struct amdgpu_device *adev)
1837 {
1838 u32 tmp = RREG32_SOC15(GC, 0, mmRLC_CNTL);
1839
1840 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0);
1841 WREG32_SOC15(GC, 0, mmRLC_CNTL, tmp);
1842 }
1843
1844 static void gfx_v10_0_rlc_reset(struct amdgpu_device *adev)
1845 {
1846 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
1847 udelay(50);
1848 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
1849 udelay(50);
1850 }
1851
1852 static void gfx_v10_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev,
1853 bool enable)
1854 {
1855 uint32_t rlc_pg_cntl;
1856
1857 rlc_pg_cntl = RREG32_SOC15(GC, 0, mmRLC_PG_CNTL);
1858
1859 if (!enable) {
1860
1861
1862
1863
1864
1865
1866
1867
1868 rlc_pg_cntl |= 0x800000;
1869 } else
1870 rlc_pg_cntl &= ~0x800000;
1871 WREG32_SOC15(GC, 0, mmRLC_PG_CNTL, rlc_pg_cntl);
1872 }
1873
1874 static void gfx_v10_0_rlc_start(struct amdgpu_device *adev)
1875 {
1876
1877
1878 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK))
1879 gfx_v10_0_rlc_smu_handshake_cntl(adev, false);
1880
1881 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
1882 udelay(50);
1883 }
1884
1885 static void gfx_v10_0_rlc_enable_srm(struct amdgpu_device *adev)
1886 {
1887 uint32_t tmp;
1888
1889
1890 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
1891 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
1892 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK;
1893 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
1894 }
1895
1896 static int gfx_v10_0_rlc_load_microcode(struct amdgpu_device *adev)
1897 {
1898 const struct rlc_firmware_header_v2_0 *hdr;
1899 const __le32 *fw_data;
1900 unsigned i, fw_size;
1901
1902 if (!adev->gfx.rlc_fw)
1903 return -EINVAL;
1904
1905 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1906 amdgpu_ucode_print_rlc_hdr(&hdr->header);
1907
1908 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
1909 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1910 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1911
1912 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
1913 RLCG_UCODE_LOADING_START_ADDRESS);
1914
1915 for (i = 0; i < fw_size; i++)
1916 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA,
1917 le32_to_cpup(fw_data++));
1918
1919 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
1920
1921 return 0;
1922 }
1923
1924 static int gfx_v10_0_rlc_resume(struct amdgpu_device *adev)
1925 {
1926 int r;
1927
1928 if (amdgpu_sriov_vf(adev))
1929 return 0;
1930
1931 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1932 r = gfx_v10_0_wait_for_rlc_autoload_complete(adev);
1933 if (r)
1934 return r;
1935
1936 r = gfx_v10_0_init_pg(adev);
1937 if (r)
1938 return r;
1939
1940
1941 gfx_v10_0_rlc_enable_srm(adev);
1942
1943 } else {
1944 adev->gfx.rlc.funcs->stop(adev);
1945
1946
1947 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
1948
1949
1950 WREG32_SOC15(GC, 0, mmRLC_PG_CNTL, 0);
1951
1952 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
1953
1954 r = gfx_v10_0_rlc_load_microcode(adev);
1955 if (r)
1956 return r;
1957 } else if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1958
1959 r = gfx_v10_0_rlc_backdoor_autoload_enable(adev);
1960 if (r)
1961 return r;
1962 }
1963
1964 r = gfx_v10_0_init_pg(adev);
1965 if (r)
1966 return r;
1967
1968 adev->gfx.rlc.funcs->start(adev);
1969
1970 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
1971 r = gfx_v10_0_wait_for_rlc_autoload_complete(adev);
1972 if (r)
1973 return r;
1974 }
1975 }
1976 return 0;
1977 }
1978
1979 static struct {
1980 FIRMWARE_ID id;
1981 unsigned int offset;
1982 unsigned int size;
1983 } rlc_autoload_info[FIRMWARE_ID_MAX];
1984
1985 static int gfx_v10_0_parse_rlc_toc(struct amdgpu_device *adev)
1986 {
1987 int ret;
1988 RLC_TABLE_OF_CONTENT *rlc_toc;
1989
1990 ret = amdgpu_bo_create_reserved(adev, adev->psp.toc_bin_size, PAGE_SIZE,
1991 AMDGPU_GEM_DOMAIN_GTT,
1992 &adev->gfx.rlc.rlc_toc_bo,
1993 &adev->gfx.rlc.rlc_toc_gpu_addr,
1994 (void **)&adev->gfx.rlc.rlc_toc_buf);
1995 if (ret) {
1996 dev_err(adev->dev, "(%d) failed to create rlc toc bo\n", ret);
1997 return ret;
1998 }
1999
2000
2001 memcpy(adev->gfx.rlc.rlc_toc_buf, adev->psp.toc_start_addr, adev->psp.toc_bin_size);
2002
2003 rlc_toc = (RLC_TABLE_OF_CONTENT *)adev->gfx.rlc.rlc_toc_buf;
2004 while (rlc_toc && (rlc_toc->id > FIRMWARE_ID_INVALID) &&
2005 (rlc_toc->id < FIRMWARE_ID_MAX)) {
2006 if ((rlc_toc->id >= FIRMWARE_ID_CP_CE) &&
2007 (rlc_toc->id <= FIRMWARE_ID_CP_MES)) {
2008
2009 rlc_toc->offset = ALIGN(rlc_toc->offset * 4, PAGE_SIZE);
2010 }
2011
2012 rlc_autoload_info[rlc_toc->id].id = rlc_toc->id;
2013 rlc_autoload_info[rlc_toc->id].offset = rlc_toc->offset * 4;
2014 rlc_autoload_info[rlc_toc->id].size = rlc_toc->size * 4;
2015
2016 rlc_toc++;
2017 };
2018
2019 return 0;
2020 }
2021
2022 static uint32_t gfx_v10_0_calc_toc_total_size(struct amdgpu_device *adev)
2023 {
2024 uint32_t total_size = 0;
2025 FIRMWARE_ID id;
2026 int ret;
2027
2028 ret = gfx_v10_0_parse_rlc_toc(adev);
2029 if (ret) {
2030 dev_err(adev->dev, "failed to parse rlc toc\n");
2031 return 0;
2032 }
2033
2034 for (id = FIRMWARE_ID_RLC_G_UCODE; id < FIRMWARE_ID_MAX; id++)
2035 total_size += rlc_autoload_info[id].size;
2036
2037
2038 if (total_size < rlc_autoload_info[FIRMWARE_ID_MAX-1].offset)
2039 total_size = rlc_autoload_info[FIRMWARE_ID_MAX-1].offset +
2040 rlc_autoload_info[FIRMWARE_ID_MAX-1].size;
2041
2042 return total_size;
2043 }
2044
2045 static int gfx_v10_0_rlc_backdoor_autoload_buffer_init(struct amdgpu_device *adev)
2046 {
2047 int r;
2048 uint32_t total_size;
2049
2050 total_size = gfx_v10_0_calc_toc_total_size(adev);
2051
2052 r = amdgpu_bo_create_reserved(adev, total_size, PAGE_SIZE,
2053 AMDGPU_GEM_DOMAIN_GTT,
2054 &adev->gfx.rlc.rlc_autoload_bo,
2055 &adev->gfx.rlc.rlc_autoload_gpu_addr,
2056 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
2057 if (r) {
2058 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r);
2059 return r;
2060 }
2061
2062 return 0;
2063 }
2064
2065 static void gfx_v10_0_rlc_backdoor_autoload_buffer_fini(struct amdgpu_device *adev)
2066 {
2067 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_toc_bo,
2068 &adev->gfx.rlc.rlc_toc_gpu_addr,
2069 (void **)&adev->gfx.rlc.rlc_toc_buf);
2070 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo,
2071 &adev->gfx.rlc.rlc_autoload_gpu_addr,
2072 (void **)&adev->gfx.rlc.rlc_autoload_ptr);
2073 }
2074
2075 static void gfx_v10_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev,
2076 FIRMWARE_ID id,
2077 const void *fw_data,
2078 uint32_t fw_size)
2079 {
2080 uint32_t toc_offset;
2081 uint32_t toc_fw_size;
2082 char *ptr = adev->gfx.rlc.rlc_autoload_ptr;
2083
2084 if (id <= FIRMWARE_ID_INVALID || id >= FIRMWARE_ID_MAX)
2085 return;
2086
2087 toc_offset = rlc_autoload_info[id].offset;
2088 toc_fw_size = rlc_autoload_info[id].size;
2089
2090 if (fw_size == 0)
2091 fw_size = toc_fw_size;
2092
2093 if (fw_size > toc_fw_size)
2094 fw_size = toc_fw_size;
2095
2096 memcpy(ptr + toc_offset, fw_data, fw_size);
2097
2098 if (fw_size < toc_fw_size)
2099 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size);
2100 }
2101
2102 static void gfx_v10_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev)
2103 {
2104 void *data;
2105 uint32_t size;
2106
2107 data = adev->gfx.rlc.rlc_toc_buf;
2108 size = rlc_autoload_info[FIRMWARE_ID_RLC_TOC].size;
2109
2110 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2111 FIRMWARE_ID_RLC_TOC,
2112 data, size);
2113 }
2114
2115 static void gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev)
2116 {
2117 const __le32 *fw_data;
2118 uint32_t fw_size;
2119 const struct gfx_firmware_header_v1_0 *cp_hdr;
2120 const struct rlc_firmware_header_v2_0 *rlc_hdr;
2121
2122
2123 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
2124 adev->gfx.pfp_fw->data;
2125 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2126 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
2127 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
2128 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2129 FIRMWARE_ID_CP_PFP,
2130 fw_data, fw_size);
2131
2132
2133 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
2134 adev->gfx.ce_fw->data;
2135 fw_data = (const __le32 *)(adev->gfx.ce_fw->data +
2136 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
2137 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
2138 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2139 FIRMWARE_ID_CP_CE,
2140 fw_data, fw_size);
2141
2142
2143 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
2144 adev->gfx.me_fw->data;
2145 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
2146 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
2147 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
2148 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2149 FIRMWARE_ID_CP_ME,
2150 fw_data, fw_size);
2151
2152
2153 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)
2154 adev->gfx.rlc_fw->data;
2155 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2156 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes));
2157 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes);
2158 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2159 FIRMWARE_ID_RLC_G_UCODE,
2160 fw_data, fw_size);
2161
2162
2163 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
2164 adev->gfx.mec_fw->data;
2165 fw_data = (const __le32 *) (adev->gfx.mec_fw->data +
2166 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes));
2167 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
2168 cp_hdr->jt_size * 4;
2169 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2170 FIRMWARE_ID_CP_MEC,
2171 fw_data, fw_size);
2172
2173 }
2174
2175
2176 static void gfx_v10_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev)
2177 {
2178 const __le32 *fw_data;
2179 uint32_t fw_size;
2180 const struct sdma_firmware_header_v1_0 *sdma_hdr;
2181 int i;
2182
2183 for (i = 0; i < adev->sdma.num_instances; i++) {
2184 sdma_hdr = (const struct sdma_firmware_header_v1_0 *)
2185 adev->sdma.instance[i].fw->data;
2186 fw_data = (const __le32 *) (adev->sdma.instance[i].fw->data +
2187 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes));
2188 fw_size = le32_to_cpu(sdma_hdr->header.ucode_size_bytes);
2189
2190 if (i == 0) {
2191 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2192 FIRMWARE_ID_SDMA0_UCODE, fw_data, fw_size);
2193 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2194 FIRMWARE_ID_SDMA0_JT,
2195 (uint32_t *)fw_data +
2196 sdma_hdr->jt_offset,
2197 sdma_hdr->jt_size * 4);
2198 } else if (i == 1) {
2199 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2200 FIRMWARE_ID_SDMA1_UCODE, fw_data, fw_size);
2201 gfx_v10_0_rlc_backdoor_autoload_copy_ucode(adev,
2202 FIRMWARE_ID_SDMA1_JT,
2203 (uint32_t *)fw_data +
2204 sdma_hdr->jt_offset,
2205 sdma_hdr->jt_size * 4);
2206 }
2207 }
2208 }
2209
2210 static int gfx_v10_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
2211 {
2212 uint32_t rlc_g_offset, rlc_g_size, tmp;
2213 uint64_t gpu_addr;
2214
2215 gfx_v10_0_rlc_backdoor_autoload_copy_toc_ucode(adev);
2216 gfx_v10_0_rlc_backdoor_autoload_copy_sdma_ucode(adev);
2217 gfx_v10_0_rlc_backdoor_autoload_copy_gfx_ucode(adev);
2218
2219 rlc_g_offset = rlc_autoload_info[FIRMWARE_ID_RLC_G_UCODE].offset;
2220 rlc_g_size = rlc_autoload_info[FIRMWARE_ID_RLC_G_UCODE].size;
2221 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset;
2222
2223 WREG32_SOC15(GC, 0, mmRLC_HYP_BOOTLOAD_ADDR_HI, upper_32_bits(gpu_addr));
2224 WREG32_SOC15(GC, 0, mmRLC_HYP_BOOTLOAD_ADDR_LO, lower_32_bits(gpu_addr));
2225 WREG32_SOC15(GC, 0, mmRLC_HYP_BOOTLOAD_SIZE, rlc_g_size);
2226
2227 tmp = RREG32_SOC15(GC, 0, mmRLC_HYP_RESET_VECTOR);
2228 if (!(tmp & (RLC_HYP_RESET_VECTOR__COLD_BOOT_EXIT_MASK |
2229 RLC_HYP_RESET_VECTOR__VDDGFX_EXIT_MASK))) {
2230 DRM_ERROR("Neither COLD_BOOT_EXIT nor VDDGFX_EXIT is set\n");
2231 return -EINVAL;
2232 }
2233
2234 tmp = RREG32_SOC15(GC, 0, mmRLC_CNTL);
2235 if (tmp & RLC_CNTL__RLC_ENABLE_F32_MASK) {
2236 DRM_ERROR("RLC ROM should halt itself\n");
2237 return -EINVAL;
2238 }
2239
2240 return 0;
2241 }
2242
2243 static int gfx_v10_0_rlc_backdoor_autoload_config_me_cache(struct amdgpu_device *adev)
2244 {
2245 uint32_t usec_timeout = 50000;
2246 uint32_t tmp;
2247 int i;
2248 uint64_t addr;
2249
2250
2251 tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL);
2252 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2253 WREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL, tmp);
2254
2255
2256 for (i = 0; i < usec_timeout; i++) {
2257 tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL);
2258 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2259 INVALIDATE_CACHE_COMPLETE))
2260 break;
2261 udelay(1);
2262 }
2263
2264 if (i >= usec_timeout) {
2265 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2266 return -EINVAL;
2267 }
2268
2269
2270 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2271 rlc_autoload_info[FIRMWARE_ID_CP_ME].offset;
2272 WREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_LO,
2273 lower_32_bits(addr) & 0xFFFFF000);
2274 WREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_HI,
2275 upper_32_bits(addr));
2276
2277 return 0;
2278 }
2279
2280 static int gfx_v10_0_rlc_backdoor_autoload_config_ce_cache(struct amdgpu_device *adev)
2281 {
2282 uint32_t usec_timeout = 50000;
2283 uint32_t tmp;
2284 int i;
2285 uint64_t addr;
2286
2287
2288 tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL);
2289 tmp = REG_SET_FIELD(tmp, CP_CE_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2290 WREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL, tmp);
2291
2292
2293 for (i = 0; i < usec_timeout; i++) {
2294 tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL);
2295 if (1 == REG_GET_FIELD(tmp, CP_CE_IC_OP_CNTL,
2296 INVALIDATE_CACHE_COMPLETE))
2297 break;
2298 udelay(1);
2299 }
2300
2301 if (i >= usec_timeout) {
2302 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2303 return -EINVAL;
2304 }
2305
2306
2307 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2308 rlc_autoload_info[FIRMWARE_ID_CP_CE].offset;
2309 WREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_LO,
2310 lower_32_bits(addr) & 0xFFFFF000);
2311 WREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_HI,
2312 upper_32_bits(addr));
2313
2314 return 0;
2315 }
2316
2317 static int gfx_v10_0_rlc_backdoor_autoload_config_pfp_cache(struct amdgpu_device *adev)
2318 {
2319 uint32_t usec_timeout = 50000;
2320 uint32_t tmp;
2321 int i;
2322 uint64_t addr;
2323
2324
2325 tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL);
2326 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2327 WREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL, tmp);
2328
2329
2330 for (i = 0; i < usec_timeout; i++) {
2331 tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL);
2332 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2333 INVALIDATE_CACHE_COMPLETE))
2334 break;
2335 udelay(1);
2336 }
2337
2338 if (i >= usec_timeout) {
2339 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2340 return -EINVAL;
2341 }
2342
2343
2344 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2345 rlc_autoload_info[FIRMWARE_ID_CP_PFP].offset;
2346 WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_LO,
2347 lower_32_bits(addr) & 0xFFFFF000);
2348 WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_HI,
2349 upper_32_bits(addr));
2350
2351 return 0;
2352 }
2353
2354 static int gfx_v10_0_rlc_backdoor_autoload_config_mec_cache(struct amdgpu_device *adev)
2355 {
2356 uint32_t usec_timeout = 50000;
2357 uint32_t tmp;
2358 int i;
2359 uint64_t addr;
2360
2361
2362 tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL);
2363 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2364 WREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL, tmp);
2365
2366
2367 for (i = 0; i < usec_timeout; i++) {
2368 tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL);
2369 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2370 INVALIDATE_CACHE_COMPLETE))
2371 break;
2372 udelay(1);
2373 }
2374
2375 if (i >= usec_timeout) {
2376 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2377 return -EINVAL;
2378 }
2379
2380
2381 addr = adev->gfx.rlc.rlc_autoload_gpu_addr +
2382 rlc_autoload_info[FIRMWARE_ID_CP_MEC].offset;
2383 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
2384 lower_32_bits(addr) & 0xFFFFF000);
2385 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
2386 upper_32_bits(addr));
2387
2388 return 0;
2389 }
2390
2391 static int gfx_v10_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
2392 {
2393 uint32_t cp_status;
2394 uint32_t bootload_status;
2395 int i, r;
2396
2397 for (i = 0; i < adev->usec_timeout; i++) {
2398 cp_status = RREG32_SOC15(GC, 0, mmCP_STAT);
2399 bootload_status = RREG32_SOC15(GC, 0, mmRLC_RLCS_BOOTLOAD_STATUS);
2400 if ((cp_status == 0) &&
2401 (REG_GET_FIELD(bootload_status,
2402 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) {
2403 break;
2404 }
2405 udelay(1);
2406 }
2407
2408 if (i >= adev->usec_timeout) {
2409 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n");
2410 return -ETIMEDOUT;
2411 }
2412
2413 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) {
2414 r = gfx_v10_0_rlc_backdoor_autoload_config_me_cache(adev);
2415 if (r)
2416 return r;
2417
2418 r = gfx_v10_0_rlc_backdoor_autoload_config_ce_cache(adev);
2419 if (r)
2420 return r;
2421
2422 r = gfx_v10_0_rlc_backdoor_autoload_config_pfp_cache(adev);
2423 if (r)
2424 return r;
2425
2426 r = gfx_v10_0_rlc_backdoor_autoload_config_mec_cache(adev);
2427 if (r)
2428 return r;
2429 }
2430
2431 return 0;
2432 }
2433
2434 static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2435 {
2436 int i;
2437 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
2438
2439 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2440 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2441 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
2442 if (!enable) {
2443 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2444 adev->gfx.gfx_ring[i].sched.ready = false;
2445 }
2446 WREG32_SOC15(GC, 0, mmCP_ME_CNTL, tmp);
2447
2448 for (i = 0; i < adev->usec_timeout; i++) {
2449 if (RREG32_SOC15(GC, 0, mmCP_STAT) == 0)
2450 break;
2451 udelay(1);
2452 }
2453
2454 if (i >= adev->usec_timeout)
2455 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt");
2456
2457 return 0;
2458 }
2459
2460 static int gfx_v10_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev)
2461 {
2462 int r;
2463 const struct gfx_firmware_header_v1_0 *pfp_hdr;
2464 const __le32 *fw_data;
2465 unsigned i, fw_size;
2466 uint32_t tmp;
2467 uint32_t usec_timeout = 50000;
2468
2469 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2470 adev->gfx.pfp_fw->data;
2471
2472 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2473
2474 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data +
2475 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
2476 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes);
2477
2478 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes,
2479 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2480 &adev->gfx.pfp.pfp_fw_obj,
2481 &adev->gfx.pfp.pfp_fw_gpu_addr,
2482 (void **)&adev->gfx.pfp.pfp_fw_ptr);
2483 if (r) {
2484 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r);
2485 gfx_v10_0_pfp_fini(adev);
2486 return r;
2487 }
2488
2489 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size);
2490
2491 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj);
2492 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj);
2493
2494
2495 tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL);
2496 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2497 WREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL, tmp);
2498
2499
2500 for (i = 0; i < usec_timeout; i++) {
2501 tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_OP_CNTL);
2502 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL,
2503 INVALIDATE_CACHE_COMPLETE))
2504 break;
2505 udelay(1);
2506 }
2507
2508 if (i >= usec_timeout) {
2509 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2510 return -EINVAL;
2511 }
2512
2513 if (amdgpu_emu_mode == 1)
2514 adev->nbio_funcs->hdp_flush(adev, NULL);
2515
2516 tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_CNTL);
2517 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0);
2518 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0);
2519 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0);
2520 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2521 WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_CNTL, tmp);
2522 WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_LO,
2523 adev->gfx.pfp.pfp_fw_gpu_addr & 0xFFFFF000);
2524 WREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_HI,
2525 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr));
2526
2527 return 0;
2528 }
2529
2530 static int gfx_v10_0_cp_gfx_load_ce_microcode(struct amdgpu_device *adev)
2531 {
2532 int r;
2533 const struct gfx_firmware_header_v1_0 *ce_hdr;
2534 const __le32 *fw_data;
2535 unsigned i, fw_size;
2536 uint32_t tmp;
2537 uint32_t usec_timeout = 50000;
2538
2539 ce_hdr = (const struct gfx_firmware_header_v1_0 *)
2540 adev->gfx.ce_fw->data;
2541
2542 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
2543
2544 fw_data = (const __le32 *)(adev->gfx.ce_fw->data +
2545 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
2546 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes);
2547
2548 r = amdgpu_bo_create_reserved(adev, ce_hdr->header.ucode_size_bytes,
2549 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2550 &adev->gfx.ce.ce_fw_obj,
2551 &adev->gfx.ce.ce_fw_gpu_addr,
2552 (void **)&adev->gfx.ce.ce_fw_ptr);
2553 if (r) {
2554 dev_err(adev->dev, "(%d) failed to create ce fw bo\n", r);
2555 gfx_v10_0_ce_fini(adev);
2556 return r;
2557 }
2558
2559 memcpy(adev->gfx.ce.ce_fw_ptr, fw_data, fw_size);
2560
2561 amdgpu_bo_kunmap(adev->gfx.ce.ce_fw_obj);
2562 amdgpu_bo_unreserve(adev->gfx.ce.ce_fw_obj);
2563
2564
2565 tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL);
2566 tmp = REG_SET_FIELD(tmp, CP_CE_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2567 WREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL, tmp);
2568
2569
2570 for (i = 0; i < usec_timeout; i++) {
2571 tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_OP_CNTL);
2572 if (1 == REG_GET_FIELD(tmp, CP_CE_IC_OP_CNTL,
2573 INVALIDATE_CACHE_COMPLETE))
2574 break;
2575 udelay(1);
2576 }
2577
2578 if (i >= usec_timeout) {
2579 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2580 return -EINVAL;
2581 }
2582
2583 if (amdgpu_emu_mode == 1)
2584 adev->nbio_funcs->hdp_flush(adev, NULL);
2585
2586 tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_CNTL);
2587 tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, VMID, 0);
2588 tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, CACHE_POLICY, 0);
2589 tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, EXE_DISABLE, 0);
2590 tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2591 WREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_LO,
2592 adev->gfx.ce.ce_fw_gpu_addr & 0xFFFFF000);
2593 WREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_HI,
2594 upper_32_bits(adev->gfx.ce.ce_fw_gpu_addr));
2595
2596 return 0;
2597 }
2598
2599 static int gfx_v10_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev)
2600 {
2601 int r;
2602 const struct gfx_firmware_header_v1_0 *me_hdr;
2603 const __le32 *fw_data;
2604 unsigned i, fw_size;
2605 uint32_t tmp;
2606 uint32_t usec_timeout = 50000;
2607
2608 me_hdr = (const struct gfx_firmware_header_v1_0 *)
2609 adev->gfx.me_fw->data;
2610
2611 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2612
2613 fw_data = (const __le32 *)(adev->gfx.me_fw->data +
2614 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
2615 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes);
2616
2617 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes,
2618 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
2619 &adev->gfx.me.me_fw_obj,
2620 &adev->gfx.me.me_fw_gpu_addr,
2621 (void **)&adev->gfx.me.me_fw_ptr);
2622 if (r) {
2623 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r);
2624 gfx_v10_0_me_fini(adev);
2625 return r;
2626 }
2627
2628 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size);
2629
2630 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj);
2631 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj);
2632
2633
2634 tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL);
2635 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2636 WREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL, tmp);
2637
2638
2639 for (i = 0; i < usec_timeout; i++) {
2640 tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_OP_CNTL);
2641 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL,
2642 INVALIDATE_CACHE_COMPLETE))
2643 break;
2644 udelay(1);
2645 }
2646
2647 if (i >= usec_timeout) {
2648 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2649 return -EINVAL;
2650 }
2651
2652 if (amdgpu_emu_mode == 1)
2653 adev->nbio_funcs->hdp_flush(adev, NULL);
2654
2655 tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_CNTL);
2656 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0);
2657 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0);
2658 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0);
2659 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2660 WREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_LO,
2661 adev->gfx.me.me_fw_gpu_addr & 0xFFFFF000);
2662 WREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_HI,
2663 upper_32_bits(adev->gfx.me.me_fw_gpu_addr));
2664
2665 return 0;
2666 }
2667
2668 static int gfx_v10_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2669 {
2670 int r;
2671
2672 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
2673 return -EINVAL;
2674
2675 gfx_v10_0_cp_gfx_enable(adev, false);
2676
2677 r = gfx_v10_0_cp_gfx_load_pfp_microcode(adev);
2678 if (r) {
2679 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r);
2680 return r;
2681 }
2682
2683 r = gfx_v10_0_cp_gfx_load_ce_microcode(adev);
2684 if (r) {
2685 dev_err(adev->dev, "(%d) failed to load ce fw\n", r);
2686 return r;
2687 }
2688
2689 r = gfx_v10_0_cp_gfx_load_me_microcode(adev);
2690 if (r) {
2691 dev_err(adev->dev, "(%d) failed to load me fw\n", r);
2692 return r;
2693 }
2694
2695 return 0;
2696 }
2697
2698 static int gfx_v10_0_cp_gfx_start(struct amdgpu_device *adev)
2699 {
2700 struct amdgpu_ring *ring;
2701 const struct cs_section_def *sect = NULL;
2702 const struct cs_extent_def *ext = NULL;
2703 int r, i;
2704 int ctx_reg_offset;
2705
2706
2707 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT,
2708 adev->gfx.config.max_hw_contexts - 1);
2709 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
2710
2711 gfx_v10_0_cp_gfx_enable(adev, true);
2712
2713 ring = &adev->gfx.gfx_ring[0];
2714 r = amdgpu_ring_alloc(ring, gfx_v10_0_get_csb_size(adev) + 4);
2715 if (r) {
2716 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
2717 return r;
2718 }
2719
2720 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2721 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2722
2723 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
2724 amdgpu_ring_write(ring, 0x80000000);
2725 amdgpu_ring_write(ring, 0x80000000);
2726
2727 for (sect = gfx10_cs_data; sect->section != NULL; ++sect) {
2728 for (ext = sect->section; ext->extent != NULL; ++ext) {
2729 if (sect->id == SECT_CONTEXT) {
2730 amdgpu_ring_write(ring,
2731 PACKET3(PACKET3_SET_CONTEXT_REG,
2732 ext->reg_count));
2733 amdgpu_ring_write(ring, ext->reg_index -
2734 PACKET3_SET_CONTEXT_REG_START);
2735 for (i = 0; i < ext->reg_count; i++)
2736 amdgpu_ring_write(ring, ext->extent[i]);
2737 }
2738 }
2739 }
2740
2741 ctx_reg_offset =
2742 SOC15_REG_OFFSET(GC, 0, mmPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START;
2743 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1));
2744 amdgpu_ring_write(ring, ctx_reg_offset);
2745 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override);
2746
2747 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2748 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2749
2750 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2751 amdgpu_ring_write(ring, 0);
2752
2753 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2754 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2755 amdgpu_ring_write(ring, 0x8000);
2756 amdgpu_ring_write(ring, 0x8000);
2757
2758 amdgpu_ring_commit(ring);
2759
2760
2761 ring = &adev->gfx.gfx_ring[1];
2762 r = amdgpu_ring_alloc(ring, 2);
2763 if (r) {
2764 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
2765 return r;
2766 }
2767
2768 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2769 amdgpu_ring_write(ring, 0);
2770
2771 amdgpu_ring_commit(ring);
2772
2773 return 0;
2774 }
2775
2776 static void gfx_v10_0_cp_gfx_switch_pipe(struct amdgpu_device *adev,
2777 CP_PIPE_ID pipe)
2778 {
2779 u32 tmp;
2780
2781 tmp = RREG32_SOC15(GC, 0, mmGRBM_GFX_CNTL);
2782 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe);
2783
2784 WREG32_SOC15(GC, 0, mmGRBM_GFX_CNTL, tmp);
2785 }
2786
2787 static void gfx_v10_0_cp_gfx_set_doorbell(struct amdgpu_device *adev,
2788 struct amdgpu_ring *ring)
2789 {
2790 u32 tmp;
2791
2792 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
2793 if (ring->use_doorbell) {
2794 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2795 DOORBELL_OFFSET, ring->doorbell_index);
2796 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2797 DOORBELL_EN, 1);
2798 } else {
2799 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
2800 DOORBELL_EN, 0);
2801 }
2802 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
2803 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
2804 DOORBELL_RANGE_LOWER, ring->doorbell_index);
2805 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
2806
2807 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
2808 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
2809 }
2810
2811 static int gfx_v10_0_cp_gfx_resume(struct amdgpu_device *adev)
2812 {
2813 struct amdgpu_ring *ring;
2814 u32 tmp;
2815 u32 rb_bufsz;
2816 u64 rb_addr, rptr_addr, wptr_gpu_addr;
2817 u32 i;
2818
2819
2820 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
2821
2822
2823 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
2824
2825
2826 mutex_lock(&adev->srbm_mutex);
2827 gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
2828 mutex_unlock(&adev->srbm_mutex);
2829
2830 ring = &adev->gfx.gfx_ring[0];
2831 rb_bufsz = order_base_2(ring->ring_size / 8);
2832 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
2833 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
2834 #ifdef __BIG_ENDIAN
2835 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
2836 #endif
2837 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
2838
2839
2840 ring->wptr = 0;
2841 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
2842 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
2843
2844
2845 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
2846 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
2847 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
2848 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
2849
2850 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
2851 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO,
2852 lower_32_bits(wptr_gpu_addr));
2853 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI,
2854 upper_32_bits(wptr_gpu_addr));
2855
2856 mdelay(1);
2857 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
2858
2859 rb_addr = ring->gpu_addr >> 8;
2860 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
2861 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
2862
2863 WREG32_SOC15(GC, 0, mmCP_RB_ACTIVE, 1);
2864
2865 gfx_v10_0_cp_gfx_set_doorbell(adev, ring);
2866
2867
2868 mutex_lock(&adev->srbm_mutex);
2869 gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
2870 mutex_unlock(&adev->srbm_mutex);
2871 ring = &adev->gfx.gfx_ring[1];
2872 rb_bufsz = order_base_2(ring->ring_size / 8);
2873 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
2874 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2);
2875 WREG32_SOC15(GC, 0, mmCP_RB1_CNTL, tmp);
2876
2877 ring->wptr = 0;
2878 WREG32_SOC15(GC, 0, mmCP_RB1_WPTR, lower_32_bits(ring->wptr));
2879 WREG32_SOC15(GC, 0, mmCP_RB1_WPTR_HI, upper_32_bits(ring->wptr));
2880
2881 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
2882 WREG32_SOC15(GC, 0, mmCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr));
2883 WREG32_SOC15(GC, 0, mmCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) &
2884 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
2885 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
2886 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO,
2887 lower_32_bits(wptr_gpu_addr));
2888 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI,
2889 upper_32_bits(wptr_gpu_addr));
2890
2891 mdelay(1);
2892 WREG32_SOC15(GC, 0, mmCP_RB1_CNTL, tmp);
2893
2894 rb_addr = ring->gpu_addr >> 8;
2895 WREG32_SOC15(GC, 0, mmCP_RB1_BASE, rb_addr);
2896 WREG32_SOC15(GC, 0, mmCP_RB1_BASE_HI, upper_32_bits(rb_addr));
2897 WREG32_SOC15(GC, 0, mmCP_RB1_ACTIVE, 1);
2898
2899 gfx_v10_0_cp_gfx_set_doorbell(adev, ring);
2900
2901
2902 mutex_lock(&adev->srbm_mutex);
2903 gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
2904 mutex_unlock(&adev->srbm_mutex);
2905
2906
2907 gfx_v10_0_cp_gfx_start(adev);
2908
2909 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2910 ring = &adev->gfx.gfx_ring[i];
2911 ring->sched.ready = true;
2912 }
2913
2914 return 0;
2915 }
2916
2917 static void gfx_v10_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
2918 {
2919 int i;
2920
2921 if (enable) {
2922 WREG32_SOC15(GC, 0, mmCP_MEC_CNTL, 0);
2923 } else {
2924 WREG32_SOC15(GC, 0, mmCP_MEC_CNTL,
2925 (CP_MEC_CNTL__MEC_ME1_HALT_MASK |
2926 CP_MEC_CNTL__MEC_ME2_HALT_MASK));
2927 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2928 adev->gfx.compute_ring[i].sched.ready = false;
2929 adev->gfx.kiq.ring.sched.ready = false;
2930 }
2931 udelay(50);
2932 }
2933
2934 static int gfx_v10_0_cp_compute_load_microcode(struct amdgpu_device *adev)
2935 {
2936 const struct gfx_firmware_header_v1_0 *mec_hdr;
2937 const __le32 *fw_data;
2938 unsigned i;
2939 u32 tmp;
2940 u32 usec_timeout = 50000;
2941
2942 if (!adev->gfx.mec_fw)
2943 return -EINVAL;
2944
2945 gfx_v10_0_cp_compute_enable(adev, false);
2946
2947 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
2948 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
2949
2950 fw_data = (const __le32 *)
2951 (adev->gfx.mec_fw->data +
2952 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
2953
2954
2955 tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL);
2956 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1);
2957 WREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL, tmp);
2958
2959
2960 for (i = 0; i < usec_timeout; i++) {
2961 tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_OP_CNTL);
2962 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL,
2963 INVALIDATE_CACHE_COMPLETE))
2964 break;
2965 udelay(1);
2966 }
2967
2968 if (i >= usec_timeout) {
2969 dev_err(adev->dev, "failed to invalidate instruction cache\n");
2970 return -EINVAL;
2971 }
2972
2973 if (amdgpu_emu_mode == 1)
2974 adev->nbio_funcs->hdp_flush(adev, NULL);
2975
2976 tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL);
2977 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
2978 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0);
2979 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1);
2980 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
2981
2982 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr &
2983 0xFFFFF000);
2984 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
2985 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
2986
2987
2988 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 0);
2989
2990 for (i = 0; i < mec_hdr->jt_size; i++)
2991 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
2992 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
2993
2994 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
2995
2996
2997
2998
2999
3000
3001 return 0;
3002 }
3003
3004 static void gfx_v10_0_kiq_setting(struct amdgpu_ring *ring)
3005 {
3006 uint32_t tmp;
3007 struct amdgpu_device *adev = ring->adev;
3008
3009
3010 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3011 tmp &= 0xffffff00;
3012 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3013 WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3014 tmp |= 0x80;
3015 WREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3016 }
3017
3018 static int gfx_v10_0_gfx_mqd_init(struct amdgpu_ring *ring)
3019 {
3020 struct amdgpu_device *adev = ring->adev;
3021 struct v10_gfx_mqd *mqd = ring->mqd_ptr;
3022 uint64_t hqd_gpu_addr, wb_gpu_addr;
3023 uint32_t tmp;
3024 uint32_t rb_bufsz;
3025
3026
3027 mqd->cp_gfx_hqd_wptr = 0;
3028 mqd->cp_gfx_hqd_wptr_hi = 0;
3029
3030
3031 mqd->cp_mqd_base_addr = ring->mqd_gpu_addr & 0xfffffffc;
3032 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3033
3034
3035 tmp = RREG32_SOC15(GC, 0, mmCP_GFX_MQD_CONTROL);
3036 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0);
3037 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1);
3038 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0);
3039 mqd->cp_gfx_mqd_control = tmp;
3040
3041
3042 tmp = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_VMID);
3043 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0);
3044 mqd->cp_gfx_hqd_vmid = 0;
3045
3046
3047
3048 tmp = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_QUEUE_PRIORITY);
3049 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, 0);
3050 mqd->cp_gfx_hqd_queue_priority = tmp;
3051
3052
3053 tmp = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_QUANTUM);
3054 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1);
3055 mqd->cp_gfx_hqd_quantum = tmp;
3056
3057
3058 hqd_gpu_addr = ring->gpu_addr >> 8;
3059 mqd->cp_gfx_hqd_base = hqd_gpu_addr;
3060 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr);
3061
3062
3063 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3064 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc;
3065 mqd->cp_gfx_hqd_rptr_addr_hi =
3066 upper_32_bits(wb_gpu_addr) & 0xffff;
3067
3068
3069 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3070 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3071 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3072
3073
3074 rb_bufsz = order_base_2(ring->ring_size / 4) - 1;
3075 tmp = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_CNTL);
3076 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz);
3077 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2);
3078 #ifdef __BIG_ENDIAN
3079 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1);
3080 #endif
3081 mqd->cp_gfx_hqd_cntl = tmp;
3082
3083
3084 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3085 if (ring->use_doorbell) {
3086 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3087 DOORBELL_OFFSET, ring->doorbell_index);
3088 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3089 DOORBELL_EN, 1);
3090 } else
3091 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3092 DOORBELL_EN, 0);
3093 mqd->cp_rb_doorbell_control = tmp;
3094
3095
3096 ring->wptr = 0;
3097 mqd->cp_gfx_hqd_rptr = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_RPTR);
3098
3099
3100 mqd->cp_gfx_hqd_active = 1;
3101
3102 return 0;
3103 }
3104
3105 #ifdef BRING_UP_DEBUG
3106 static int gfx_v10_0_gfx_queue_init_register(struct amdgpu_ring *ring)
3107 {
3108 struct amdgpu_device *adev = ring->adev;
3109 struct v10_gfx_mqd *mqd = ring->mqd_ptr;
3110
3111
3112 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_WPTR, mqd->cp_gfx_hqd_wptr);
3113 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_WPTR_HI, mqd->cp_gfx_hqd_wptr_hi);
3114
3115
3116 WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr);
3117 WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
3118
3119
3120 WREG32_SOC15(GC, 0, mmCP_GFX_MQD_CONTROL, mqd->cp_gfx_mqd_control);
3121
3122
3123 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_VMID, mqd->cp_gfx_hqd_vmid);
3124
3125 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_QUEUE_PRIORITY,
3126 mqd->cp_gfx_hqd_queue_priority);
3127 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_QUANTUM, mqd->cp_gfx_hqd_quantum);
3128
3129
3130 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_BASE, mqd->cp_gfx_hqd_base);
3131 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_BASE_HI, mqd->cp_gfx_hqd_base_hi);
3132
3133
3134 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_RPTR_ADDR, mqd->cp_gfx_hqd_rptr_addr);
3135 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_RPTR_ADDR_HI, mqd->cp_gfx_hqd_rptr_addr_hi);
3136
3137
3138 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_CNTL, mqd->cp_gfx_hqd_cntl);
3139
3140
3141 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, mqd->cp_rb_wptr_poll_addr_lo);
3142 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, mqd->cp_rb_wptr_poll_addr_hi);
3143
3144
3145 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, mqd->cp_rb_doorbell_control);
3146
3147
3148 WREG32_SOC15(GC, 0, mmCP_GFX_HQD_ACTIVE, mqd->cp_gfx_hqd_active);
3149
3150 return 0;
3151 }
3152 #endif
3153
3154 static int gfx_v10_0_gfx_init_queue(struct amdgpu_ring *ring)
3155 {
3156 struct amdgpu_device *adev = ring->adev;
3157 struct v10_gfx_mqd *mqd = ring->mqd_ptr;
3158
3159 if (!adev->in_gpu_reset && !adev->in_suspend) {
3160 memset((void *)mqd, 0, sizeof(*mqd));
3161 mutex_lock(&adev->srbm_mutex);
3162 nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3163 gfx_v10_0_gfx_mqd_init(ring);
3164 #ifdef BRING_UP_DEBUG
3165 gfx_v10_0_gfx_queue_init_register(ring);
3166 #endif
3167 nv_grbm_select(adev, 0, 0, 0, 0);
3168 mutex_unlock(&adev->srbm_mutex);
3169 if (adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS])
3170 memcpy(adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS], mqd, sizeof(*mqd));
3171 } else if (adev->in_gpu_reset) {
3172
3173 if (adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS])
3174 memcpy(mqd, adev->gfx.me.mqd_backup[AMDGPU_MAX_GFX_RINGS], sizeof(*mqd));
3175
3176 ring->wptr = 0;
3177 amdgpu_ring_clear_ring(ring);
3178 #ifdef BRING_UP_DEBUG
3179 mutex_lock(&adev->srbm_mutex);
3180 nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3181 gfx_v10_0_gfx_queue_init_register(ring);
3182 nv_grbm_select(adev, 0, 0, 0, 0);
3183 mutex_unlock(&adev->srbm_mutex);
3184 #endif
3185 } else {
3186 amdgpu_ring_clear_ring(ring);
3187 }
3188
3189 return 0;
3190 }
3191
3192 #ifndef BRING_UP_DEBUG
3193 static int gfx_v10_0_kiq_enable_kgq(struct amdgpu_device *adev)
3194 {
3195 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
3196 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
3197 int r, i;
3198
3199 if (!kiq->pmf || !kiq->pmf->kiq_map_queues)
3200 return -EINVAL;
3201
3202 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size *
3203 adev->gfx.num_gfx_rings);
3204 if (r) {
3205 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
3206 return r;
3207 }
3208
3209 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3210 kiq->pmf->kiq_map_queues(kiq_ring, &adev->gfx.gfx_ring[i]);
3211
3212 r = amdgpu_ring_test_ring(kiq_ring);
3213 if (r) {
3214 DRM_ERROR("kfq enable failed\n");
3215 kiq_ring->sched.ready = false;
3216 }
3217 return r;
3218 }
3219 #endif
3220
3221 static int gfx_v10_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev)
3222 {
3223 int r, i;
3224 struct amdgpu_ring *ring;
3225
3226 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3227 ring = &adev->gfx.gfx_ring[i];
3228
3229 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3230 if (unlikely(r != 0))
3231 goto done;
3232
3233 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3234 if (!r) {
3235 r = gfx_v10_0_gfx_init_queue(ring);
3236 amdgpu_bo_kunmap(ring->mqd_obj);
3237 ring->mqd_ptr = NULL;
3238 }
3239 amdgpu_bo_unreserve(ring->mqd_obj);
3240 if (r)
3241 goto done;
3242 }
3243 #ifndef BRING_UP_DEBUG
3244 r = gfx_v10_0_kiq_enable_kgq(adev);
3245 if (r)
3246 goto done;
3247 #endif
3248 r = gfx_v10_0_cp_gfx_start(adev);
3249 if (r)
3250 goto done;
3251
3252 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3253 ring = &adev->gfx.gfx_ring[i];
3254 ring->sched.ready = true;
3255 }
3256 done:
3257 return r;
3258 }
3259
3260 static int gfx_v10_0_compute_mqd_init(struct amdgpu_ring *ring)
3261 {
3262 struct amdgpu_device *adev = ring->adev;
3263 struct v10_compute_mqd *mqd = ring->mqd_ptr;
3264 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3265 uint32_t tmp;
3266
3267 mqd->header = 0xC0310800;
3268 mqd->compute_pipelinestat_enable = 0x00000001;
3269 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3270 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3271 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3272 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3273 mqd->compute_misc_reserved = 0x00000003;
3274
3275 eop_base_addr = ring->eop_gpu_addr >> 8;
3276 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3277 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3278
3279
3280 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3281 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3282 (order_base_2(GFX10_MEC_HPD_SIZE / 4) - 1));
3283
3284 mqd->cp_hqd_eop_control = tmp;
3285
3286
3287 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3288
3289 if (ring->use_doorbell) {
3290 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3291 DOORBELL_OFFSET, ring->doorbell_index);
3292 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3293 DOORBELL_EN, 1);
3294 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3295 DOORBELL_SOURCE, 0);
3296 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3297 DOORBELL_HIT, 0);
3298 } else {
3299 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3300 DOORBELL_EN, 0);
3301 }
3302
3303 mqd->cp_hqd_pq_doorbell_control = tmp;
3304
3305
3306 ring->wptr = 0;
3307 mqd->cp_hqd_dequeue_request = 0;
3308 mqd->cp_hqd_pq_rptr = 0;
3309 mqd->cp_hqd_pq_wptr_lo = 0;
3310 mqd->cp_hqd_pq_wptr_hi = 0;
3311
3312
3313 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3314 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3315
3316
3317 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3318 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3319 mqd->cp_mqd_control = tmp;
3320
3321
3322 hqd_gpu_addr = ring->gpu_addr >> 8;
3323 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3324 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3325
3326
3327 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3328 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3329 (order_base_2(ring->ring_size / 4) - 1));
3330 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3331 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3332 #ifdef __BIG_ENDIAN
3333 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3334 #endif
3335 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3336 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0);
3337 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3338 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3339 mqd->cp_hqd_pq_control = tmp;
3340
3341
3342 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3343 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3344 mqd->cp_hqd_pq_rptr_report_addr_hi =
3345 upper_32_bits(wb_gpu_addr) & 0xffff;
3346
3347
3348 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3349 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3350 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3351
3352 tmp = 0;
3353
3354 if (ring->use_doorbell) {
3355 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3356 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3357 DOORBELL_OFFSET, ring->doorbell_index);
3358
3359 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3360 DOORBELL_EN, 1);
3361 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3362 DOORBELL_SOURCE, 0);
3363 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3364 DOORBELL_HIT, 0);
3365 }
3366
3367 mqd->cp_hqd_pq_doorbell_control = tmp;
3368
3369
3370 ring->wptr = 0;
3371 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3372
3373
3374 mqd->cp_hqd_vmid = 0;
3375
3376 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3377 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3378 mqd->cp_hqd_persistent_state = tmp;
3379
3380
3381 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3382 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3383 mqd->cp_hqd_ib_control = tmp;
3384
3385
3386 mqd->cp_hqd_active = 1;
3387
3388 return 0;
3389 }
3390
3391 static int gfx_v10_0_kiq_init_register(struct amdgpu_ring *ring)
3392 {
3393 struct amdgpu_device *adev = ring->adev;
3394 struct v10_compute_mqd *mqd = ring->mqd_ptr;
3395 int j;
3396
3397
3398 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3399
3400
3401 WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3402 mqd->cp_hqd_eop_base_addr_lo);
3403 WREG32_SOC15(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3404 mqd->cp_hqd_eop_base_addr_hi);
3405
3406
3407 WREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL,
3408 mqd->cp_hqd_eop_control);
3409
3410
3411 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3412 mqd->cp_hqd_pq_doorbell_control);
3413
3414
3415 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3416 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3417 for (j = 0; j < adev->usec_timeout; j++) {
3418 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3419 break;
3420 udelay(1);
3421 }
3422 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3423 mqd->cp_hqd_dequeue_request);
3424 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR,
3425 mqd->cp_hqd_pq_rptr);
3426 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3427 mqd->cp_hqd_pq_wptr_lo);
3428 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3429 mqd->cp_hqd_pq_wptr_hi);
3430 }
3431
3432
3433 WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR,
3434 mqd->cp_mqd_base_addr_lo);
3435 WREG32_SOC15(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3436 mqd->cp_mqd_base_addr_hi);
3437
3438
3439 WREG32_SOC15(GC, 0, mmCP_MQD_CONTROL,
3440 mqd->cp_mqd_control);
3441
3442
3443 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE,
3444 mqd->cp_hqd_pq_base_lo);
3445 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_BASE_HI,
3446 mqd->cp_hqd_pq_base_hi);
3447
3448
3449 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL,
3450 mqd->cp_hqd_pq_control);
3451
3452
3453 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3454 mqd->cp_hqd_pq_rptr_report_addr_lo);
3455 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3456 mqd->cp_hqd_pq_rptr_report_addr_hi);
3457
3458
3459 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3460 mqd->cp_hqd_pq_wptr_poll_addr_lo);
3461 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3462 mqd->cp_hqd_pq_wptr_poll_addr_hi);
3463
3464
3465 if (ring->use_doorbell) {
3466 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3467 (adev->doorbell_index.kiq * 2) << 2);
3468 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3469 (adev->doorbell_index.userqueue_end * 2) << 2);
3470 }
3471
3472 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3473 mqd->cp_hqd_pq_doorbell_control);
3474
3475
3476 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3477 mqd->cp_hqd_pq_wptr_lo);
3478 WREG32_SOC15(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3479 mqd->cp_hqd_pq_wptr_hi);
3480
3481
3482 WREG32_SOC15(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3483
3484 WREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3485 mqd->cp_hqd_persistent_state);
3486
3487
3488 WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE,
3489 mqd->cp_hqd_active);
3490
3491 if (ring->use_doorbell)
3492 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3493
3494 return 0;
3495 }
3496
3497 static int gfx_v10_0_kiq_init_queue(struct amdgpu_ring *ring)
3498 {
3499 struct amdgpu_device *adev = ring->adev;
3500 struct v10_compute_mqd *mqd = ring->mqd_ptr;
3501 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3502
3503 gfx_v10_0_kiq_setting(ring);
3504
3505 if (adev->in_gpu_reset) {
3506
3507 if (adev->gfx.mec.mqd_backup[mqd_idx])
3508 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
3509
3510
3511 ring->wptr = 0;
3512 amdgpu_ring_clear_ring(ring);
3513
3514 mutex_lock(&adev->srbm_mutex);
3515 nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3516 gfx_v10_0_kiq_init_register(ring);
3517 nv_grbm_select(adev, 0, 0, 0, 0);
3518 mutex_unlock(&adev->srbm_mutex);
3519 } else {
3520 memset((void *)mqd, 0, sizeof(*mqd));
3521 mutex_lock(&adev->srbm_mutex);
3522 nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3523 gfx_v10_0_compute_mqd_init(ring);
3524 gfx_v10_0_kiq_init_register(ring);
3525 nv_grbm_select(adev, 0, 0, 0, 0);
3526 mutex_unlock(&adev->srbm_mutex);
3527
3528 if (adev->gfx.mec.mqd_backup[mqd_idx])
3529 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3530 }
3531
3532 return 0;
3533 }
3534
3535 static int gfx_v10_0_kcq_init_queue(struct amdgpu_ring *ring)
3536 {
3537 struct amdgpu_device *adev = ring->adev;
3538 struct v10_compute_mqd *mqd = ring->mqd_ptr;
3539 int mqd_idx = ring - &adev->gfx.compute_ring[0];
3540
3541 if (!adev->in_gpu_reset && !adev->in_suspend) {
3542 memset((void *)mqd, 0, sizeof(*mqd));
3543 mutex_lock(&adev->srbm_mutex);
3544 nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3545 gfx_v10_0_compute_mqd_init(ring);
3546 nv_grbm_select(adev, 0, 0, 0, 0);
3547 mutex_unlock(&adev->srbm_mutex);
3548
3549 if (adev->gfx.mec.mqd_backup[mqd_idx])
3550 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd));
3551 } else if (adev->in_gpu_reset) {
3552
3553 if (adev->gfx.mec.mqd_backup[mqd_idx])
3554 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd));
3555
3556
3557 ring->wptr = 0;
3558 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3559 amdgpu_ring_clear_ring(ring);
3560 } else {
3561 amdgpu_ring_clear_ring(ring);
3562 }
3563
3564 return 0;
3565 }
3566
3567 static int gfx_v10_0_kiq_resume(struct amdgpu_device *adev)
3568 {
3569 struct amdgpu_ring *ring;
3570 int r;
3571
3572 ring = &adev->gfx.kiq.ring;
3573
3574 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3575 if (unlikely(r != 0))
3576 return r;
3577
3578 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3579 if (unlikely(r != 0))
3580 return r;
3581
3582 gfx_v10_0_kiq_init_queue(ring);
3583 amdgpu_bo_kunmap(ring->mqd_obj);
3584 ring->mqd_ptr = NULL;
3585 amdgpu_bo_unreserve(ring->mqd_obj);
3586 ring->sched.ready = true;
3587 return 0;
3588 }
3589
3590 static int gfx_v10_0_kcq_resume(struct amdgpu_device *adev)
3591 {
3592 struct amdgpu_ring *ring = NULL;
3593 int r = 0, i;
3594
3595 gfx_v10_0_cp_compute_enable(adev, true);
3596
3597 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3598 ring = &adev->gfx.compute_ring[i];
3599
3600 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3601 if (unlikely(r != 0))
3602 goto done;
3603 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3604 if (!r) {
3605 r = gfx_v10_0_kcq_init_queue(ring);
3606 amdgpu_bo_kunmap(ring->mqd_obj);
3607 ring->mqd_ptr = NULL;
3608 }
3609 amdgpu_bo_unreserve(ring->mqd_obj);
3610 if (r)
3611 goto done;
3612 }
3613
3614 r = amdgpu_gfx_enable_kcq(adev);
3615 done:
3616 return r;
3617 }
3618
3619 static int gfx_v10_0_cp_resume(struct amdgpu_device *adev)
3620 {
3621 int r, i;
3622 struct amdgpu_ring *ring;
3623
3624 if (!(adev->flags & AMD_IS_APU))
3625 gfx_v10_0_enable_gui_idle_interrupt(adev, false);
3626
3627 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
3628
3629 r = gfx_v10_0_cp_gfx_load_microcode(adev);
3630 if (r)
3631 return r;
3632
3633 r = gfx_v10_0_cp_compute_load_microcode(adev);
3634 if (r)
3635 return r;
3636 }
3637
3638 r = gfx_v10_0_kiq_resume(adev);
3639 if (r)
3640 return r;
3641
3642 r = gfx_v10_0_kcq_resume(adev);
3643 if (r)
3644 return r;
3645
3646 if (!amdgpu_async_gfx_ring) {
3647 r = gfx_v10_0_cp_gfx_resume(adev);
3648 if (r)
3649 return r;
3650 } else {
3651 r = gfx_v10_0_cp_async_gfx_ring_resume(adev);
3652 if (r)
3653 return r;
3654 }
3655
3656 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
3657 ring = &adev->gfx.gfx_ring[i];
3658 DRM_INFO("gfx %d ring me %d pipe %d q %d\n",
3659 i, ring->me, ring->pipe, ring->queue);
3660 r = amdgpu_ring_test_ring(ring);
3661 if (r) {
3662 ring->sched.ready = false;
3663 return r;
3664 }
3665 }
3666
3667 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3668 ring = &adev->gfx.compute_ring[i];
3669 ring->sched.ready = true;
3670 DRM_INFO("compute ring %d mec %d pipe %d q %d\n",
3671 i, ring->me, ring->pipe, ring->queue);
3672 r = amdgpu_ring_test_ring(ring);
3673 if (r)
3674 ring->sched.ready = false;
3675 }
3676
3677 return 0;
3678 }
3679
3680 static void gfx_v10_0_cp_enable(struct amdgpu_device *adev, bool enable)
3681 {
3682 gfx_v10_0_cp_gfx_enable(adev, enable);
3683 gfx_v10_0_cp_compute_enable(adev, enable);
3684 }
3685
3686 static bool gfx_v10_0_check_grbm_cam_remapping(struct amdgpu_device *adev)
3687 {
3688 uint32_t data, pattern = 0xDEADBEEF;
3689
3690
3691
3692 data = RREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE);
3693
3694 WREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE, 0);
3695
3696 WREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE_UMD, pattern);
3697
3698 if (RREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE) == pattern) {
3699 WREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE_UMD, data);
3700 return true;
3701 } else {
3702 WREG32_SOC15(GC, 0, mmVGT_ESGS_RING_SIZE, data);
3703 return false;
3704 }
3705 }
3706
3707 static void gfx_v10_0_setup_grbm_cam_remapping(struct amdgpu_device *adev)
3708 {
3709 uint32_t data;
3710
3711
3712
3713 WREG32_SOC15(GC, 0, mmGRBM_CAM_INDEX, 0);
3714
3715
3716 data = (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_RING_SIZE_UMD) <<
3717 GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
3718 (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_RING_SIZE) <<
3719 GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
3720 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
3721 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
3722
3723
3724 data = (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_MEMORY_BASE_UMD) <<
3725 GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
3726 (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_MEMORY_BASE) <<
3727 GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
3728 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
3729 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
3730
3731
3732 data = (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_MEMORY_BASE_HI_UMD) <<
3733 GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
3734 (SOC15_REG_OFFSET(GC, 0, mmVGT_TF_MEMORY_BASE_HI) <<
3735 GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
3736 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
3737 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
3738
3739
3740 data = (SOC15_REG_OFFSET(GC, 0, mmVGT_HS_OFFCHIP_PARAM_UMD) <<
3741 GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
3742 (SOC15_REG_OFFSET(GC, 0, mmVGT_HS_OFFCHIP_PARAM) <<
3743 GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
3744 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
3745 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
3746
3747
3748 data = (SOC15_REG_OFFSET(GC, 0, mmVGT_ESGS_RING_SIZE_UMD) <<
3749 GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
3750 (SOC15_REG_OFFSET(GC, 0, mmVGT_ESGS_RING_SIZE) <<
3751 GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
3752 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
3753 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
3754
3755
3756 data = (SOC15_REG_OFFSET(GC, 0, mmVGT_GSVS_RING_SIZE_UMD) <<
3757 GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
3758 (SOC15_REG_OFFSET(GC, 0, mmVGT_GSVS_RING_SIZE) <<
3759 GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
3760 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
3761 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
3762
3763
3764 data = (SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL_REMAP) <<
3765 GRBM_CAM_DATA__CAM_ADDR__SHIFT) |
3766 (SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL) <<
3767 GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT);
3768 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA_UPPER, 0);
3769 WREG32_SOC15(GC, 0, mmGRBM_CAM_DATA, data);
3770 }
3771
3772 static int gfx_v10_0_hw_init(void *handle)
3773 {
3774 int r;
3775 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3776
3777 r = gfx_v10_0_csb_vram_pin(adev);
3778 if (r)
3779 return r;
3780
3781 if (!amdgpu_emu_mode)
3782 gfx_v10_0_init_golden_registers(adev);
3783
3784 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) {
3785
3786
3787
3788
3789
3790 r = smu_load_microcode(&adev->smu);
3791 if (r)
3792 return r;
3793
3794 r = smu_check_fw_status(&adev->smu);
3795 if (r) {
3796 pr_err("SMC firmware status is not correct\n");
3797 return r;
3798 }
3799 }
3800
3801
3802 if (!gfx_v10_0_check_grbm_cam_remapping(adev))
3803 gfx_v10_0_setup_grbm_cam_remapping(adev);
3804
3805 gfx_v10_0_constants_init(adev);
3806
3807 r = gfx_v10_0_rlc_resume(adev);
3808 if (r)
3809 return r;
3810
3811
3812
3813
3814
3815 gfx_v10_0_tcp_harvest(adev);
3816
3817 r = gfx_v10_0_cp_resume(adev);
3818 if (r)
3819 return r;
3820
3821 return r;
3822 }
3823
3824 #ifndef BRING_UP_DEBUG
3825 static int gfx_v10_0_kiq_disable_kgq(struct amdgpu_device *adev)
3826 {
3827 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
3828 struct amdgpu_ring *kiq_ring = &kiq->ring;
3829 int i;
3830
3831 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
3832 return -EINVAL;
3833
3834 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size *
3835 adev->gfx.num_gfx_rings))
3836 return -ENOMEM;
3837
3838 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3839 kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i],
3840 PREEMPT_QUEUES, 0, 0);
3841
3842 return amdgpu_ring_test_ring(kiq_ring);
3843 }
3844 #endif
3845
3846 static int gfx_v10_0_hw_fini(void *handle)
3847 {
3848 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3849 int r;
3850
3851 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3852 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3853 #ifndef BRING_UP_DEBUG
3854 if (amdgpu_async_gfx_ring) {
3855 r = gfx_v10_0_kiq_disable_kgq(adev);
3856 if (r)
3857 DRM_ERROR("KGQ disable failed\n");
3858 }
3859 #endif
3860 if (amdgpu_gfx_disable_kcq(adev))
3861 DRM_ERROR("KCQ disable failed\n");
3862 if (amdgpu_sriov_vf(adev)) {
3863 pr_debug("For SRIOV client, shouldn't do anything.\n");
3864 return 0;
3865 }
3866 gfx_v10_0_cp_enable(adev, false);
3867 gfx_v10_0_enable_gui_idle_interrupt(adev, false);
3868 gfx_v10_0_csb_vram_unpin(adev);
3869
3870 return 0;
3871 }
3872
3873 static int gfx_v10_0_suspend(void *handle)
3874 {
3875 return gfx_v10_0_hw_fini(handle);
3876 }
3877
3878 static int gfx_v10_0_resume(void *handle)
3879 {
3880 return gfx_v10_0_hw_init(handle);
3881 }
3882
3883 static bool gfx_v10_0_is_idle(void *handle)
3884 {
3885 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3886
3887 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3888 GRBM_STATUS, GUI_ACTIVE))
3889 return false;
3890 else
3891 return true;
3892 }
3893
3894 static int gfx_v10_0_wait_for_idle(void *handle)
3895 {
3896 unsigned i;
3897 u32 tmp;
3898 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3899
3900 for (i = 0; i < adev->usec_timeout; i++) {
3901
3902 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS) &
3903 GRBM_STATUS__GUI_ACTIVE_MASK;
3904
3905 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE))
3906 return 0;
3907 udelay(1);
3908 }
3909 return -ETIMEDOUT;
3910 }
3911
3912 static int gfx_v10_0_soft_reset(void *handle)
3913 {
3914 u32 grbm_soft_reset = 0;
3915 u32 tmp;
3916 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3917
3918
3919 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
3920 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
3921 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
3922 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__DB_BUSY_MASK |
3923 GRBM_STATUS__CB_BUSY_MASK | GRBM_STATUS__GDS_BUSY_MASK |
3924 GRBM_STATUS__SPI_BUSY_MASK | GRBM_STATUS__GE_BUSY_NO_DMA_MASK
3925 | GRBM_STATUS__BCI_BUSY_MASK)) {
3926 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3927 GRBM_SOFT_RESET, SOFT_RESET_CP,
3928 1);
3929 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3930 GRBM_SOFT_RESET, SOFT_RESET_GFX,
3931 1);
3932 }
3933
3934 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
3935 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3936 GRBM_SOFT_RESET, SOFT_RESET_CP,
3937 1);
3938 }
3939
3940
3941 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
3942 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
3943 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3944 GRBM_SOFT_RESET, SOFT_RESET_RLC,
3945 1);
3946
3947 if (grbm_soft_reset) {
3948
3949 gfx_v10_0_rlc_stop(adev);
3950
3951
3952 gfx_v10_0_cp_gfx_enable(adev, false);
3953
3954
3955 gfx_v10_0_cp_compute_enable(adev, false);
3956
3957 if (grbm_soft_reset) {
3958 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3959 tmp |= grbm_soft_reset;
3960 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3961 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3962 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3963
3964 udelay(50);
3965
3966 tmp &= ~grbm_soft_reset;
3967 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3968 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3969 }
3970
3971
3972 udelay(50);
3973 }
3974 return 0;
3975 }
3976
3977 static uint64_t gfx_v10_0_get_gpu_clock_counter(struct amdgpu_device *adev)
3978 {
3979 uint64_t clock;
3980
3981 amdgpu_gfx_off_ctrl(adev, false);
3982 mutex_lock(&adev->gfx.gpu_clock_mutex);
3983 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
3984 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
3985 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
3986 mutex_unlock(&adev->gfx.gpu_clock_mutex);
3987 amdgpu_gfx_off_ctrl(adev, true);
3988 return clock;
3989 }
3990
3991 static void gfx_v10_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
3992 uint32_t vmid,
3993 uint32_t gds_base, uint32_t gds_size,
3994 uint32_t gws_base, uint32_t gws_size,
3995 uint32_t oa_base, uint32_t oa_size)
3996 {
3997 struct amdgpu_device *adev = ring->adev;
3998
3999
4000 gfx_v10_0_write_data_to_reg(ring, 0, false,
4001 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4002 gds_base);
4003
4004
4005 gfx_v10_0_write_data_to_reg(ring, 0, false,
4006 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4007 gds_size);
4008
4009
4010 gfx_v10_0_write_data_to_reg(ring, 0, false,
4011 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4012 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4013
4014
4015 gfx_v10_0_write_data_to_reg(ring, 0, false,
4016 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4017 (1 << (oa_size + oa_base)) - (1 << oa_base));
4018 }
4019
4020 static int gfx_v10_0_early_init(void *handle)
4021 {
4022 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4023
4024 adev->gfx.num_gfx_rings = GFX10_NUM_GFX_RINGS;
4025 adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
4026
4027 gfx_v10_0_set_kiq_pm4_funcs(adev);
4028 gfx_v10_0_set_ring_funcs(adev);
4029 gfx_v10_0_set_irq_funcs(adev);
4030 gfx_v10_0_set_gds_init(adev);
4031 gfx_v10_0_set_rlc_funcs(adev);
4032
4033 return 0;
4034 }
4035
4036 static int gfx_v10_0_late_init(void *handle)
4037 {
4038 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4039 int r;
4040
4041 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4042 if (r)
4043 return r;
4044
4045 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4046 if (r)
4047 return r;
4048
4049 return 0;
4050 }
4051
4052 static bool gfx_v10_0_is_rlc_enabled(struct amdgpu_device *adev)
4053 {
4054 uint32_t rlc_cntl;
4055
4056
4057 rlc_cntl = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4058 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false;
4059 }
4060
4061 static void gfx_v10_0_set_safe_mode(struct amdgpu_device *adev)
4062 {
4063 uint32_t data;
4064 unsigned i;
4065
4066 data = RLC_SAFE_MODE__CMD_MASK;
4067 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4068 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4069
4070
4071 for (i = 0; i < adev->usec_timeout; i++) {
4072 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4073 break;
4074 udelay(1);
4075 }
4076 }
4077
4078 static void gfx_v10_0_unset_safe_mode(struct amdgpu_device *adev)
4079 {
4080 uint32_t data;
4081
4082 data = RLC_SAFE_MODE__CMD_MASK;
4083 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4084 }
4085
4086 static void gfx_v10_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4087 bool enable)
4088 {
4089 uint32_t data, def;
4090
4091
4092 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4093
4094 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4095 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4096 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4097 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4098
4099
4100 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4101
4102 if (def != data)
4103 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4104
4105
4106 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4107
4108 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4109 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4110 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4111 if (def != data)
4112 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4113 }
4114
4115 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4116 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4117 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4118 if (def != data)
4119 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4120 }
4121 }
4122 } else {
4123
4124 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4125 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4126 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4127 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4128 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4129 if (def != data)
4130 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4131
4132
4133 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4134 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4135 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4136 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4137 }
4138
4139
4140 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4141 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4142 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4143 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4144 }
4145 }
4146 }
4147
4148 static void gfx_v10_0_update_3d_clock_gating(struct amdgpu_device *adev,
4149 bool enable)
4150 {
4151 uint32_t data, def;
4152
4153
4154 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
4155
4156 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4157
4158 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4159
4160 if (def != data)
4161 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4162
4163 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4164 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4165 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4166 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4167 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4168 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4169 if (def != data)
4170 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4171
4172
4173 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4174 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4175 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4176 if (def != data)
4177 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4178 } else {
4179
4180 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4181
4182 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4183 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4184
4185 if (def != data)
4186 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4187 }
4188 }
4189
4190 static void gfx_v10_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4191 bool enable)
4192 {
4193 uint32_t def, data;
4194
4195 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4196 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4197
4198 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4199 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4200 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4201 else
4202 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4203
4204 if (def != data)
4205 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4206
4207
4208 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4209 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4210 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4211 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4212 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4213 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4214 if (def != data)
4215 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4216
4217
4218 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4219 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4220 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4221 if (def != data)
4222 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4223 } else {
4224 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4225
4226 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4227
4228 if (def != data)
4229 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4230 }
4231 }
4232
4233 static int gfx_v10_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4234 bool enable)
4235 {
4236 amdgpu_gfx_rlc_enter_safe_mode(adev);
4237
4238 if (enable) {
4239
4240
4241
4242 gfx_v10_0_update_medium_grain_clock_gating(adev, enable);
4243
4244 gfx_v10_0_update_3d_clock_gating(adev, enable);
4245
4246 gfx_v10_0_update_coarse_grain_clock_gating(adev, enable);
4247 } else {
4248
4249
4250
4251 gfx_v10_0_update_coarse_grain_clock_gating(adev, enable);
4252
4253 gfx_v10_0_update_3d_clock_gating(adev, enable);
4254
4255 gfx_v10_0_update_medium_grain_clock_gating(adev, enable);
4256 }
4257
4258 if (adev->cg_flags &
4259 (AMD_CG_SUPPORT_GFX_MGCG |
4260 AMD_CG_SUPPORT_GFX_CGLS |
4261 AMD_CG_SUPPORT_GFX_CGCG |
4262 AMD_CG_SUPPORT_GFX_CGLS |
4263 AMD_CG_SUPPORT_GFX_3D_CGCG |
4264 AMD_CG_SUPPORT_GFX_3D_CGLS))
4265 gfx_v10_0_enable_gui_idle_interrupt(adev, enable);
4266
4267 amdgpu_gfx_rlc_exit_safe_mode(adev);
4268
4269 return 0;
4270 }
4271
4272 static const struct amdgpu_rlc_funcs gfx_v10_0_rlc_funcs = {
4273 .is_rlc_enabled = gfx_v10_0_is_rlc_enabled,
4274 .set_safe_mode = gfx_v10_0_set_safe_mode,
4275 .unset_safe_mode = gfx_v10_0_unset_safe_mode,
4276 .init = gfx_v10_0_rlc_init,
4277 .get_csb_size = gfx_v10_0_get_csb_size,
4278 .get_csb_buffer = gfx_v10_0_get_csb_buffer,
4279 .resume = gfx_v10_0_rlc_resume,
4280 .stop = gfx_v10_0_rlc_stop,
4281 .reset = gfx_v10_0_rlc_reset,
4282 .start = gfx_v10_0_rlc_start
4283 };
4284
4285 static int gfx_v10_0_set_powergating_state(void *handle,
4286 enum amd_powergating_state state)
4287 {
4288 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4289 bool enable = (state == AMD_PG_STATE_GATE) ? true : false;
4290 switch (adev->asic_type) {
4291 case CHIP_NAVI10:
4292 case CHIP_NAVI14:
4293 amdgpu_gfx_off_ctrl(adev, enable);
4294 break;
4295 default:
4296 break;
4297 }
4298 return 0;
4299 }
4300
4301 static int gfx_v10_0_set_clockgating_state(void *handle,
4302 enum amd_clockgating_state state)
4303 {
4304 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4305
4306 switch (adev->asic_type) {
4307 case CHIP_NAVI10:
4308 case CHIP_NAVI14:
4309 case CHIP_NAVI12:
4310 gfx_v10_0_update_gfx_clock_gating(adev,
4311 state == AMD_CG_STATE_GATE ? true : false);
4312 break;
4313 default:
4314 break;
4315 }
4316 return 0;
4317 }
4318
4319 static void gfx_v10_0_get_clockgating_state(void *handle, u32 *flags)
4320 {
4321 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4322 int data;
4323
4324
4325 data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4326 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
4327 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
4328
4329
4330 data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4331 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
4332 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
4333
4334
4335 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
4336 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
4337
4338
4339 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4340 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
4341 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
4342
4343
4344 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4345 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
4346 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
4347
4348
4349 data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4350 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
4351 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
4352
4353
4354 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
4355 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
4356 }
4357
4358 static u64 gfx_v10_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
4359 {
4360 return ring->adev->wb.wb[ring->rptr_offs];
4361 }
4362
4363 static u64 gfx_v10_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
4364 {
4365 struct amdgpu_device *adev = ring->adev;
4366 u64 wptr;
4367
4368
4369 if (ring->use_doorbell) {
4370 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
4371 } else {
4372 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
4373 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
4374 }
4375
4376 return wptr;
4377 }
4378
4379 static void gfx_v10_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
4380 {
4381 struct amdgpu_device *adev = ring->adev;
4382
4383 if (ring->use_doorbell) {
4384
4385 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
4386 WDOORBELL64(ring->doorbell_index, ring->wptr);
4387 } else {
4388 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
4389 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
4390 }
4391 }
4392
4393 static u64 gfx_v10_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
4394 {
4395 return ring->adev->wb.wb[ring->rptr_offs];
4396 }
4397
4398 static u64 gfx_v10_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
4399 {
4400 u64 wptr;
4401
4402
4403 if (ring->use_doorbell)
4404 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
4405 else
4406 BUG();
4407 return wptr;
4408 }
4409
4410 static void gfx_v10_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
4411 {
4412 struct amdgpu_device *adev = ring->adev;
4413
4414
4415 if (ring->use_doorbell) {
4416 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr);
4417 WDOORBELL64(ring->doorbell_index, ring->wptr);
4418 } else {
4419 BUG();
4420 }
4421 }
4422
4423 static void gfx_v10_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
4424 {
4425 struct amdgpu_device *adev = ring->adev;
4426 u32 ref_and_mask, reg_mem_engine;
4427 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio_funcs->hdp_flush_reg;
4428
4429 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
4430 switch (ring->me) {
4431 case 1:
4432 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
4433 break;
4434 case 2:
4435 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
4436 break;
4437 default:
4438 return;
4439 }
4440 reg_mem_engine = 0;
4441 } else {
4442 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
4443 reg_mem_engine = 1;
4444 }
4445
4446 gfx_v10_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
4447 adev->nbio_funcs->get_hdp_flush_req_offset(adev),
4448 adev->nbio_funcs->get_hdp_flush_done_offset(adev),
4449 ref_and_mask, ref_and_mask, 0x20);
4450 }
4451
4452 static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
4453 struct amdgpu_job *job,
4454 struct amdgpu_ib *ib,
4455 uint32_t flags)
4456 {
4457 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
4458 u32 header, control = 0;
4459
4460 if (ib->flags & AMDGPU_IB_FLAG_CE)
4461 header = PACKET3(PACKET3_INDIRECT_BUFFER_CNST, 2);
4462 else
4463 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4464
4465 control |= ib->length_dw | (vmid << 24);
4466
4467 if (amdgpu_mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
4468 control |= INDIRECT_BUFFER_PRE_ENB(1);
4469
4470 if (flags & AMDGPU_IB_PREEMPTED)
4471 control |= INDIRECT_BUFFER_PRE_RESUME(1);
4472
4473 if (!(ib->flags & AMDGPU_IB_FLAG_CE))
4474 gfx_v10_0_ring_emit_de_meta(ring,
4475 flags & AMDGPU_IB_PREEMPTED ? true : false);
4476 }
4477
4478 amdgpu_ring_write(ring, header);
4479 BUG_ON(ib->gpu_addr & 0x3);
4480 amdgpu_ring_write(ring,
4481 #ifdef __BIG_ENDIAN
4482 (2 << 0) |
4483 #endif
4484 lower_32_bits(ib->gpu_addr));
4485 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
4486 amdgpu_ring_write(ring, control);
4487 }
4488
4489 static void gfx_v10_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
4490 struct amdgpu_job *job,
4491 struct amdgpu_ib *ib,
4492 uint32_t flags)
4493 {
4494 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
4495 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
4508 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
4509 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
4510 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
4511 }
4512
4513 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
4514 BUG_ON(ib->gpu_addr & 0x3);
4515 amdgpu_ring_write(ring,
4516 #ifdef __BIG_ENDIAN
4517 (2 << 0) |
4518 #endif
4519 lower_32_bits(ib->gpu_addr));
4520 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
4521 amdgpu_ring_write(ring, control);
4522 }
4523
4524 static void gfx_v10_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
4525 u64 seq, unsigned flags)
4526 {
4527 struct amdgpu_device *adev = ring->adev;
4528 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
4529 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
4530
4531
4532 if (adev->pdev->device == 0x50)
4533 int_sel = false;
4534
4535
4536 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
4537 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ |
4538 PACKET3_RELEASE_MEM_GCR_GL2_WB |
4539 PACKET3_RELEASE_MEM_GCR_GLM_INV |
4540 PACKET3_RELEASE_MEM_GCR_GLM_WB |
4541 PACKET3_RELEASE_MEM_CACHE_POLICY(3) |
4542 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
4543 PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
4544 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
4545 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
4546
4547
4548
4549
4550
4551 if (write64bit)
4552 BUG_ON(addr & 0x7);
4553 else
4554 BUG_ON(addr & 0x3);
4555 amdgpu_ring_write(ring, lower_32_bits(addr));
4556 amdgpu_ring_write(ring, upper_32_bits(addr));
4557 amdgpu_ring_write(ring, lower_32_bits(seq));
4558 amdgpu_ring_write(ring, upper_32_bits(seq));
4559 amdgpu_ring_write(ring, 0);
4560 }
4561
4562 static void gfx_v10_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
4563 {
4564 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
4565 uint32_t seq = ring->fence_drv.sync_seq;
4566 uint64_t addr = ring->fence_drv.gpu_addr;
4567
4568 gfx_v10_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr),
4569 upper_32_bits(addr), seq, 0xffffffff, 4);
4570 }
4571
4572 static void gfx_v10_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
4573 unsigned vmid, uint64_t pd_addr)
4574 {
4575 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
4576
4577
4578 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
4579
4580 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4581 amdgpu_ring_write(ring, 0x0);
4582 }
4583 }
4584
4585 static void gfx_v10_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
4586 u64 seq, unsigned int flags)
4587 {
4588 struct amdgpu_device *adev = ring->adev;
4589
4590
4591 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
4592
4593
4594 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4595 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4596 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
4597 amdgpu_ring_write(ring, lower_32_bits(addr));
4598 amdgpu_ring_write(ring, upper_32_bits(addr));
4599 amdgpu_ring_write(ring, lower_32_bits(seq));
4600
4601 if (flags & AMDGPU_FENCE_FLAG_INT) {
4602
4603 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4604 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4605 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
4606 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
4607 amdgpu_ring_write(ring, 0);
4608 amdgpu_ring_write(ring, 0x20000000);
4609 }
4610 }
4611
4612 static void gfx_v10_0_ring_emit_sb(struct amdgpu_ring *ring)
4613 {
4614 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4615 amdgpu_ring_write(ring, 0);
4616 }
4617
4618 static void gfx_v10_0_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
4619 {
4620 uint32_t dw2 = 0;
4621
4622 if (amdgpu_mcbp)
4623 gfx_v10_0_ring_emit_ce_meta(ring,
4624 flags & AMDGPU_IB_PREEMPTED ? true : false);
4625
4626 gfx_v10_0_ring_emit_tmz(ring, true);
4627
4628 dw2 |= 0x80000000;
4629 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
4630
4631 dw2 |= 0x8001;
4632
4633 dw2 |= 0x01000000;
4634
4635 dw2 |= 0x10002;
4636
4637
4638 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
4639 dw2 |= 0x10000000;
4640 } else {
4641
4642
4643
4644 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
4645 dw2 |= 0x10000000;
4646 }
4647
4648 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4649 amdgpu_ring_write(ring, dw2);
4650 amdgpu_ring_write(ring, 0);
4651 }
4652
4653 static unsigned gfx_v10_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
4654 {
4655 unsigned ret;
4656
4657 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
4658 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
4659 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
4660 amdgpu_ring_write(ring, 0);
4661 ret = ring->wptr & ring->buf_mask;
4662 amdgpu_ring_write(ring, 0x55aa55aa);
4663
4664 return ret;
4665 }
4666
4667 static void gfx_v10_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
4668 {
4669 unsigned cur;
4670 BUG_ON(offset > ring->buf_mask);
4671 BUG_ON(ring->ring[offset] != 0x55aa55aa);
4672
4673 cur = (ring->wptr - 1) & ring->buf_mask;
4674 if (likely(cur > offset))
4675 ring->ring[offset] = cur - offset;
4676 else
4677 ring->ring[offset] = (ring->buf_mask + 1) - offset + cur;
4678 }
4679
4680 static int gfx_v10_0_ring_preempt_ib(struct amdgpu_ring *ring)
4681 {
4682 int i, r = 0;
4683 struct amdgpu_device *adev = ring->adev;
4684 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4685 struct amdgpu_ring *kiq_ring = &kiq->ring;
4686
4687 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
4688 return -EINVAL;
4689
4690 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size))
4691 return -ENOMEM;
4692
4693
4694 amdgpu_ring_set_preempt_cond_exec(ring, false);
4695
4696
4697 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
4698 ring->trail_fence_gpu_addr,
4699 ++ring->trail_seq);
4700 amdgpu_ring_commit(kiq_ring);
4701
4702
4703 for (i = 0; i < adev->usec_timeout; i++) {
4704 if (ring->trail_seq ==
4705 le32_to_cpu(*(ring->trail_fence_cpu_addr)))
4706 break;
4707 udelay(1);
4708 }
4709
4710 if (i >= adev->usec_timeout) {
4711 r = -EINVAL;
4712 DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
4713 }
4714
4715
4716 amdgpu_ring_set_preempt_cond_exec(ring, true);
4717 return r;
4718 }
4719
4720 static void gfx_v10_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume)
4721 {
4722 struct amdgpu_device *adev = ring->adev;
4723 struct v10_ce_ib_state ce_payload = {0};
4724 uint64_t csa_addr;
4725 int cnt;
4726
4727 cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
4728 csa_addr = amdgpu_csa_vaddr(ring->adev);
4729
4730 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
4731 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
4732 WRITE_DATA_DST_SEL(8) |
4733 WR_CONFIRM) |
4734 WRITE_DATA_CACHE_POLICY(0));
4735 amdgpu_ring_write(ring, lower_32_bits(csa_addr +
4736 offsetof(struct v10_gfx_meta_data, ce_payload)));
4737 amdgpu_ring_write(ring, upper_32_bits(csa_addr +
4738 offsetof(struct v10_gfx_meta_data, ce_payload)));
4739
4740 if (resume)
4741 amdgpu_ring_write_multiple(ring, adev->virt.csa_cpu_addr +
4742 offsetof(struct v10_gfx_meta_data,
4743 ce_payload),
4744 sizeof(ce_payload) >> 2);
4745 else
4746 amdgpu_ring_write_multiple(ring, (void *)&ce_payload,
4747 sizeof(ce_payload) >> 2);
4748 }
4749
4750 static void gfx_v10_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
4751 {
4752 struct amdgpu_device *adev = ring->adev;
4753 struct v10_de_ib_state de_payload = {0};
4754 uint64_t csa_addr, gds_addr;
4755 int cnt;
4756
4757 csa_addr = amdgpu_csa_vaddr(ring->adev);
4758 gds_addr = ALIGN(csa_addr + AMDGPU_CSA_SIZE - adev->gds.gds_size,
4759 PAGE_SIZE);
4760 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
4761 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
4762
4763 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
4764 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
4765 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
4766 WRITE_DATA_DST_SEL(8) |
4767 WR_CONFIRM) |
4768 WRITE_DATA_CACHE_POLICY(0));
4769 amdgpu_ring_write(ring, lower_32_bits(csa_addr +
4770 offsetof(struct v10_gfx_meta_data, de_payload)));
4771 amdgpu_ring_write(ring, upper_32_bits(csa_addr +
4772 offsetof(struct v10_gfx_meta_data, de_payload)));
4773
4774 if (resume)
4775 amdgpu_ring_write_multiple(ring, adev->virt.csa_cpu_addr +
4776 offsetof(struct v10_gfx_meta_data,
4777 de_payload),
4778 sizeof(de_payload) >> 2);
4779 else
4780 amdgpu_ring_write_multiple(ring, (void *)&de_payload,
4781 sizeof(de_payload) >> 2);
4782 }
4783
4784 static void gfx_v10_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
4785 {
4786 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
4787 amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1));
4788 }
4789
4790 static void gfx_v10_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
4791 {
4792 struct amdgpu_device *adev = ring->adev;
4793
4794 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4795 amdgpu_ring_write(ring, 0 |
4796 (5 << 8) |
4797 (1 << 20));
4798 amdgpu_ring_write(ring, reg);
4799 amdgpu_ring_write(ring, 0);
4800 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4801 adev->virt.reg_val_offs * 4));
4802 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4803 adev->virt.reg_val_offs * 4));
4804 }
4805
4806 static void gfx_v10_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
4807 uint32_t val)
4808 {
4809 uint32_t cmd = 0;
4810
4811 switch (ring->funcs->type) {
4812 case AMDGPU_RING_TYPE_GFX:
4813 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
4814 break;
4815 case AMDGPU_RING_TYPE_KIQ:
4816 cmd = (1 << 16);
4817 break;
4818 default:
4819 cmd = WR_CONFIRM;
4820 break;
4821 }
4822 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4823 amdgpu_ring_write(ring, cmd);
4824 amdgpu_ring_write(ring, reg);
4825 amdgpu_ring_write(ring, 0);
4826 amdgpu_ring_write(ring, val);
4827 }
4828
4829 static void gfx_v10_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
4830 uint32_t val, uint32_t mask)
4831 {
4832 gfx_v10_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
4833 }
4834
4835 static void gfx_v10_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
4836 uint32_t reg0, uint32_t reg1,
4837 uint32_t ref, uint32_t mask)
4838 {
4839 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
4840 struct amdgpu_device *adev = ring->adev;
4841 bool fw_version_ok = false;
4842
4843 fw_version_ok = adev->gfx.cp_fw_write_wait;
4844
4845 if (fw_version_ok)
4846 gfx_v10_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
4847 ref, mask, 0x20);
4848 else
4849 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
4850 ref, mask);
4851 }
4852
4853 static void
4854 gfx_v10_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
4855 uint32_t me, uint32_t pipe,
4856 enum amdgpu_interrupt_state state)
4857 {
4858 uint32_t cp_int_cntl, cp_int_cntl_reg;
4859
4860 if (!me) {
4861 switch (pipe) {
4862 case 0:
4863 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING0);
4864 break;
4865 case 1:
4866 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_INT_CNTL_RING1);
4867 break;
4868 default:
4869 DRM_DEBUG("invalid pipe %d\n", pipe);
4870 return;
4871 }
4872 } else {
4873 DRM_DEBUG("invalid me %d\n", me);
4874 return;
4875 }
4876
4877 switch (state) {
4878 case AMDGPU_IRQ_STATE_DISABLE:
4879 cp_int_cntl = RREG32(cp_int_cntl_reg);
4880 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4881 TIME_STAMP_INT_ENABLE, 0);
4882 WREG32(cp_int_cntl_reg, cp_int_cntl);
4883 break;
4884 case AMDGPU_IRQ_STATE_ENABLE:
4885 cp_int_cntl = RREG32(cp_int_cntl_reg);
4886 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0,
4887 TIME_STAMP_INT_ENABLE, 1);
4888 WREG32(cp_int_cntl_reg, cp_int_cntl);
4889 break;
4890 default:
4891 break;
4892 }
4893 }
4894
4895 static void gfx_v10_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
4896 int me, int pipe,
4897 enum amdgpu_interrupt_state state)
4898 {
4899 u32 mec_int_cntl, mec_int_cntl_reg;
4900
4901
4902
4903
4904
4905
4906
4907 if (me == 1) {
4908 switch (pipe) {
4909 case 0:
4910 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
4911 break;
4912 case 1:
4913 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
4914 break;
4915 case 2:
4916 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
4917 break;
4918 case 3:
4919 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
4920 break;
4921 default:
4922 DRM_DEBUG("invalid pipe %d\n", pipe);
4923 return;
4924 }
4925 } else {
4926 DRM_DEBUG("invalid me %d\n", me);
4927 return;
4928 }
4929
4930 switch (state) {
4931 case AMDGPU_IRQ_STATE_DISABLE:
4932 mec_int_cntl = RREG32(mec_int_cntl_reg);
4933 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4934 TIME_STAMP_INT_ENABLE, 0);
4935 WREG32(mec_int_cntl_reg, mec_int_cntl);
4936 break;
4937 case AMDGPU_IRQ_STATE_ENABLE:
4938 mec_int_cntl = RREG32(mec_int_cntl_reg);
4939 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
4940 TIME_STAMP_INT_ENABLE, 1);
4941 WREG32(mec_int_cntl_reg, mec_int_cntl);
4942 break;
4943 default:
4944 break;
4945 }
4946 }
4947
4948 static int gfx_v10_0_set_eop_interrupt_state(struct amdgpu_device *adev,
4949 struct amdgpu_irq_src *src,
4950 unsigned type,
4951 enum amdgpu_interrupt_state state)
4952 {
4953 switch (type) {
4954 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
4955 gfx_v10_0_set_gfx_eop_interrupt_state(adev, 0, 0, state);
4956 break;
4957 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP:
4958 gfx_v10_0_set_gfx_eop_interrupt_state(adev, 0, 1, state);
4959 break;
4960 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
4961 gfx_v10_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
4962 break;
4963 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
4964 gfx_v10_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
4965 break;
4966 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
4967 gfx_v10_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
4968 break;
4969 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
4970 gfx_v10_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
4971 break;
4972 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
4973 gfx_v10_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
4974 break;
4975 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
4976 gfx_v10_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
4977 break;
4978 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
4979 gfx_v10_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
4980 break;
4981 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
4982 gfx_v10_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
4983 break;
4984 default:
4985 break;
4986 }
4987 return 0;
4988 }
4989
4990 static int gfx_v10_0_eop_irq(struct amdgpu_device *adev,
4991 struct amdgpu_irq_src *source,
4992 struct amdgpu_iv_entry *entry)
4993 {
4994 int i;
4995 u8 me_id, pipe_id, queue_id;
4996 struct amdgpu_ring *ring;
4997
4998 DRM_DEBUG("IH: CP EOP\n");
4999 me_id = (entry->ring_id & 0x0c) >> 2;
5000 pipe_id = (entry->ring_id & 0x03) >> 0;
5001 queue_id = (entry->ring_id & 0x70) >> 4;
5002
5003 switch (me_id) {
5004 case 0:
5005 if (pipe_id == 0)
5006 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5007 else
5008 amdgpu_fence_process(&adev->gfx.gfx_ring[1]);
5009 break;
5010 case 1:
5011 case 2:
5012 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5013 ring = &adev->gfx.compute_ring[i];
5014
5015
5016
5017 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5018 amdgpu_fence_process(ring);
5019 }
5020 break;
5021 }
5022 return 0;
5023 }
5024
5025 static int gfx_v10_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5026 struct amdgpu_irq_src *source,
5027 unsigned type,
5028 enum amdgpu_interrupt_state state)
5029 {
5030 switch (state) {
5031 case AMDGPU_IRQ_STATE_DISABLE:
5032 case AMDGPU_IRQ_STATE_ENABLE:
5033 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5034 PRIV_REG_INT_ENABLE,
5035 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5036 break;
5037 default:
5038 break;
5039 }
5040
5041 return 0;
5042 }
5043
5044 static int gfx_v10_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5045 struct amdgpu_irq_src *source,
5046 unsigned type,
5047 enum amdgpu_interrupt_state state)
5048 {
5049 switch (state) {
5050 case AMDGPU_IRQ_STATE_DISABLE:
5051 case AMDGPU_IRQ_STATE_ENABLE:
5052 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5053 PRIV_INSTR_INT_ENABLE,
5054 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5055 default:
5056 break;
5057 }
5058
5059 return 0;
5060 }
5061
5062 static void gfx_v10_0_handle_priv_fault(struct amdgpu_device *adev,
5063 struct amdgpu_iv_entry *entry)
5064 {
5065 u8 me_id, pipe_id, queue_id;
5066 struct amdgpu_ring *ring;
5067 int i;
5068
5069 me_id = (entry->ring_id & 0x0c) >> 2;
5070 pipe_id = (entry->ring_id & 0x03) >> 0;
5071 queue_id = (entry->ring_id & 0x70) >> 4;
5072
5073 switch (me_id) {
5074 case 0:
5075 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
5076 ring = &adev->gfx.gfx_ring[i];
5077
5078 if (ring->me == me_id && ring->pipe == pipe_id)
5079 drm_sched_fault(&ring->sched);
5080 }
5081 break;
5082 case 1:
5083 case 2:
5084 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5085 ring = &adev->gfx.compute_ring[i];
5086 if (ring->me == me_id && ring->pipe == pipe_id &&
5087 ring->queue == queue_id)
5088 drm_sched_fault(&ring->sched);
5089 }
5090 break;
5091 default:
5092 BUG();
5093 }
5094 }
5095
5096 static int gfx_v10_0_priv_reg_irq(struct amdgpu_device *adev,
5097 struct amdgpu_irq_src *source,
5098 struct amdgpu_iv_entry *entry)
5099 {
5100 DRM_ERROR("Illegal register access in command stream\n");
5101 gfx_v10_0_handle_priv_fault(adev, entry);
5102 return 0;
5103 }
5104
5105 static int gfx_v10_0_priv_inst_irq(struct amdgpu_device *adev,
5106 struct amdgpu_irq_src *source,
5107 struct amdgpu_iv_entry *entry)
5108 {
5109 DRM_ERROR("Illegal instruction in command stream\n");
5110 gfx_v10_0_handle_priv_fault(adev, entry);
5111 return 0;
5112 }
5113
5114 static int gfx_v10_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
5115 struct amdgpu_irq_src *src,
5116 unsigned int type,
5117 enum amdgpu_interrupt_state state)
5118 {
5119 uint32_t tmp, target;
5120 struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
5121
5122 if (ring->me == 1)
5123 target = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5124 else
5125 target = SOC15_REG_OFFSET(GC, 0, mmCP_ME2_PIPE0_INT_CNTL);
5126 target += ring->pipe;
5127
5128 switch (type) {
5129 case AMDGPU_CP_KIQ_IRQ_DRIVER0:
5130 if (state == AMDGPU_IRQ_STATE_DISABLE) {
5131 tmp = RREG32_SOC15(GC, 0, mmCPC_INT_CNTL);
5132 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
5133 GENERIC2_INT_ENABLE, 0);
5134 WREG32_SOC15(GC, 0, mmCPC_INT_CNTL, tmp);
5135
5136 tmp = RREG32(target);
5137 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
5138 GENERIC2_INT_ENABLE, 0);
5139 WREG32(target, tmp);
5140 } else {
5141 tmp = RREG32_SOC15(GC, 0, mmCPC_INT_CNTL);
5142 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
5143 GENERIC2_INT_ENABLE, 1);
5144 WREG32_SOC15(GC, 0, mmCPC_INT_CNTL, tmp);
5145
5146 tmp = RREG32(target);
5147 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
5148 GENERIC2_INT_ENABLE, 1);
5149 WREG32(target, tmp);
5150 }
5151 break;
5152 default:
5153 BUG();
5154 break;
5155 }
5156 return 0;
5157 }
5158
5159 static int gfx_v10_0_kiq_irq(struct amdgpu_device *adev,
5160 struct amdgpu_irq_src *source,
5161 struct amdgpu_iv_entry *entry)
5162 {
5163 u8 me_id, pipe_id, queue_id;
5164 struct amdgpu_ring *ring = &(adev->gfx.kiq.ring);
5165
5166 me_id = (entry->ring_id & 0x0c) >> 2;
5167 pipe_id = (entry->ring_id & 0x03) >> 0;
5168 queue_id = (entry->ring_id & 0x70) >> 4;
5169 DRM_DEBUG("IH: CPC GENERIC2_INT, me:%d, pipe:%d, queue:%d\n",
5170 me_id, pipe_id, queue_id);
5171
5172 amdgpu_fence_process(ring);
5173 return 0;
5174 }
5175
5176 static const struct amd_ip_funcs gfx_v10_0_ip_funcs = {
5177 .name = "gfx_v10_0",
5178 .early_init = gfx_v10_0_early_init,
5179 .late_init = gfx_v10_0_late_init,
5180 .sw_init = gfx_v10_0_sw_init,
5181 .sw_fini = gfx_v10_0_sw_fini,
5182 .hw_init = gfx_v10_0_hw_init,
5183 .hw_fini = gfx_v10_0_hw_fini,
5184 .suspend = gfx_v10_0_suspend,
5185 .resume = gfx_v10_0_resume,
5186 .is_idle = gfx_v10_0_is_idle,
5187 .wait_for_idle = gfx_v10_0_wait_for_idle,
5188 .soft_reset = gfx_v10_0_soft_reset,
5189 .set_clockgating_state = gfx_v10_0_set_clockgating_state,
5190 .set_powergating_state = gfx_v10_0_set_powergating_state,
5191 .get_clockgating_state = gfx_v10_0_get_clockgating_state,
5192 };
5193
5194 static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_gfx = {
5195 .type = AMDGPU_RING_TYPE_GFX,
5196 .align_mask = 0xff,
5197 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
5198 .support_64bit_ptrs = true,
5199 .vmhub = AMDGPU_GFXHUB_0,
5200 .get_rptr = gfx_v10_0_ring_get_rptr_gfx,
5201 .get_wptr = gfx_v10_0_ring_get_wptr_gfx,
5202 .set_wptr = gfx_v10_0_ring_set_wptr_gfx,
5203 .emit_frame_size =
5204 5 +
5205 7 +
5206 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
5207 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
5208 2 +
5209 8 +
5210 20 +
5211 4 +
5212
5213
5214
5215 5 +
5216 7 +
5217 4 +
5218 14 +
5219 31 +
5220 3 +
5221 5 +
5222 8 + 8 +
5223 2,
5224 .emit_ib_size = 4,
5225 .emit_ib = gfx_v10_0_ring_emit_ib_gfx,
5226 .emit_fence = gfx_v10_0_ring_emit_fence,
5227 .emit_pipeline_sync = gfx_v10_0_ring_emit_pipeline_sync,
5228 .emit_vm_flush = gfx_v10_0_ring_emit_vm_flush,
5229 .emit_gds_switch = gfx_v10_0_ring_emit_gds_switch,
5230 .emit_hdp_flush = gfx_v10_0_ring_emit_hdp_flush,
5231 .test_ring = gfx_v10_0_ring_test_ring,
5232 .test_ib = gfx_v10_0_ring_test_ib,
5233 .insert_nop = amdgpu_ring_insert_nop,
5234 .pad_ib = amdgpu_ring_generic_pad_ib,
5235 .emit_switch_buffer = gfx_v10_0_ring_emit_sb,
5236 .emit_cntxcntl = gfx_v10_0_ring_emit_cntxcntl,
5237 .init_cond_exec = gfx_v10_0_ring_emit_init_cond_exec,
5238 .patch_cond_exec = gfx_v10_0_ring_emit_patch_cond_exec,
5239 .preempt_ib = gfx_v10_0_ring_preempt_ib,
5240 .emit_tmz = gfx_v10_0_ring_emit_tmz,
5241 .emit_wreg = gfx_v10_0_ring_emit_wreg,
5242 .emit_reg_wait = gfx_v10_0_ring_emit_reg_wait,
5243 .emit_reg_write_reg_wait = gfx_v10_0_ring_emit_reg_write_reg_wait,
5244 };
5245
5246 static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_compute = {
5247 .type = AMDGPU_RING_TYPE_COMPUTE,
5248 .align_mask = 0xff,
5249 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
5250 .support_64bit_ptrs = true,
5251 .vmhub = AMDGPU_GFXHUB_0,
5252 .get_rptr = gfx_v10_0_ring_get_rptr_compute,
5253 .get_wptr = gfx_v10_0_ring_get_wptr_compute,
5254 .set_wptr = gfx_v10_0_ring_set_wptr_compute,
5255 .emit_frame_size =
5256 20 +
5257 7 +
5258 5 +
5259 7 +
5260 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
5261 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
5262 2 +
5263 8 + 8 + 8,
5264 .emit_ib_size = 7,
5265 .emit_ib = gfx_v10_0_ring_emit_ib_compute,
5266 .emit_fence = gfx_v10_0_ring_emit_fence,
5267 .emit_pipeline_sync = gfx_v10_0_ring_emit_pipeline_sync,
5268 .emit_vm_flush = gfx_v10_0_ring_emit_vm_flush,
5269 .emit_gds_switch = gfx_v10_0_ring_emit_gds_switch,
5270 .emit_hdp_flush = gfx_v10_0_ring_emit_hdp_flush,
5271 .test_ring = gfx_v10_0_ring_test_ring,
5272 .test_ib = gfx_v10_0_ring_test_ib,
5273 .insert_nop = amdgpu_ring_insert_nop,
5274 .pad_ib = amdgpu_ring_generic_pad_ib,
5275 .emit_wreg = gfx_v10_0_ring_emit_wreg,
5276 .emit_reg_wait = gfx_v10_0_ring_emit_reg_wait,
5277 .emit_reg_write_reg_wait = gfx_v10_0_ring_emit_reg_write_reg_wait,
5278 };
5279
5280 static const struct amdgpu_ring_funcs gfx_v10_0_ring_funcs_kiq = {
5281 .type = AMDGPU_RING_TYPE_KIQ,
5282 .align_mask = 0xff,
5283 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
5284 .support_64bit_ptrs = true,
5285 .vmhub = AMDGPU_GFXHUB_0,
5286 .get_rptr = gfx_v10_0_ring_get_rptr_compute,
5287 .get_wptr = gfx_v10_0_ring_get_wptr_compute,
5288 .set_wptr = gfx_v10_0_ring_set_wptr_compute,
5289 .emit_frame_size =
5290 20 +
5291 7 +
5292 5 +
5293 7 +
5294 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
5295 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
5296 2 +
5297 8 + 8 + 8,
5298 .emit_ib_size = 7,
5299 .emit_ib = gfx_v10_0_ring_emit_ib_compute,
5300 .emit_fence = gfx_v10_0_ring_emit_fence_kiq,
5301 .test_ring = gfx_v10_0_ring_test_ring,
5302 .test_ib = gfx_v10_0_ring_test_ib,
5303 .insert_nop = amdgpu_ring_insert_nop,
5304 .pad_ib = amdgpu_ring_generic_pad_ib,
5305 .emit_rreg = gfx_v10_0_ring_emit_rreg,
5306 .emit_wreg = gfx_v10_0_ring_emit_wreg,
5307 .emit_reg_wait = gfx_v10_0_ring_emit_reg_wait,
5308 .emit_reg_write_reg_wait = gfx_v10_0_ring_emit_reg_write_reg_wait,
5309 };
5310
5311 static void gfx_v10_0_set_ring_funcs(struct amdgpu_device *adev)
5312 {
5313 int i;
5314
5315 adev->gfx.kiq.ring.funcs = &gfx_v10_0_ring_funcs_kiq;
5316
5317 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
5318 adev->gfx.gfx_ring[i].funcs = &gfx_v10_0_ring_funcs_gfx;
5319
5320 for (i = 0; i < adev->gfx.num_compute_rings; i++)
5321 adev->gfx.compute_ring[i].funcs = &gfx_v10_0_ring_funcs_compute;
5322 }
5323
5324 static const struct amdgpu_irq_src_funcs gfx_v10_0_eop_irq_funcs = {
5325 .set = gfx_v10_0_set_eop_interrupt_state,
5326 .process = gfx_v10_0_eop_irq,
5327 };
5328
5329 static const struct amdgpu_irq_src_funcs gfx_v10_0_priv_reg_irq_funcs = {
5330 .set = gfx_v10_0_set_priv_reg_fault_state,
5331 .process = gfx_v10_0_priv_reg_irq,
5332 };
5333
5334 static const struct amdgpu_irq_src_funcs gfx_v10_0_priv_inst_irq_funcs = {
5335 .set = gfx_v10_0_set_priv_inst_fault_state,
5336 .process = gfx_v10_0_priv_inst_irq,
5337 };
5338
5339 static const struct amdgpu_irq_src_funcs gfx_v10_0_kiq_irq_funcs = {
5340 .set = gfx_v10_0_kiq_set_interrupt_state,
5341 .process = gfx_v10_0_kiq_irq,
5342 };
5343
5344 static void gfx_v10_0_set_irq_funcs(struct amdgpu_device *adev)
5345 {
5346 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
5347 adev->gfx.eop_irq.funcs = &gfx_v10_0_eop_irq_funcs;
5348
5349 adev->gfx.kiq.irq.num_types = AMDGPU_CP_KIQ_IRQ_LAST;
5350 adev->gfx.kiq.irq.funcs = &gfx_v10_0_kiq_irq_funcs;
5351
5352 adev->gfx.priv_reg_irq.num_types = 1;
5353 adev->gfx.priv_reg_irq.funcs = &gfx_v10_0_priv_reg_irq_funcs;
5354
5355 adev->gfx.priv_inst_irq.num_types = 1;
5356 adev->gfx.priv_inst_irq.funcs = &gfx_v10_0_priv_inst_irq_funcs;
5357 }
5358
5359 static void gfx_v10_0_set_rlc_funcs(struct amdgpu_device *adev)
5360 {
5361 switch (adev->asic_type) {
5362 case CHIP_NAVI10:
5363 case CHIP_NAVI14:
5364 case CHIP_NAVI12:
5365 adev->gfx.rlc.funcs = &gfx_v10_0_rlc_funcs;
5366 break;
5367 default:
5368 break;
5369 }
5370 }
5371
5372 static void gfx_v10_0_set_gds_init(struct amdgpu_device *adev)
5373 {
5374
5375 switch (adev->asic_type) {
5376 case CHIP_NAVI10:
5377 default:
5378 adev->gds.gds_size = 0x10000;
5379 adev->gds.gds_compute_max_wave_id = 0x4ff;
5380 break;
5381 }
5382
5383 adev->gds.gws_size = 64;
5384 adev->gds.oa_size = 16;
5385 }
5386
5387 static void gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev,
5388 u32 bitmap)
5389 {
5390 u32 data;
5391
5392 if (!bitmap)
5393 return;
5394
5395 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
5396 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
5397
5398 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
5399 }
5400
5401 static u32 gfx_v10_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev)
5402 {
5403 u32 data, wgp_bitmask;
5404 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
5405 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
5406
5407 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK;
5408 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT;
5409
5410 wgp_bitmask =
5411 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1);
5412
5413 return (~data) & wgp_bitmask;
5414 }
5415
5416 static u32 gfx_v10_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev)
5417 {
5418 u32 wgp_idx, wgp_active_bitmap;
5419 u32 cu_bitmap_per_wgp, cu_active_bitmap;
5420
5421 wgp_active_bitmap = gfx_v10_0_get_wgp_active_bitmap_per_sh(adev);
5422 cu_active_bitmap = 0;
5423
5424 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) {
5425
5426 cu_bitmap_per_wgp = 3 << (2 * wgp_idx);
5427 if (wgp_active_bitmap & (1 << wgp_idx))
5428 cu_active_bitmap |= cu_bitmap_per_wgp;
5429 }
5430
5431 return cu_active_bitmap;
5432 }
5433
5434 static int gfx_v10_0_get_cu_info(struct amdgpu_device *adev,
5435 struct amdgpu_cu_info *cu_info)
5436 {
5437 int i, j, k, counter, active_cu_number = 0;
5438 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
5439 unsigned disable_masks[4 * 2];
5440
5441 if (!adev || !cu_info)
5442 return -EINVAL;
5443
5444 amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
5445
5446 mutex_lock(&adev->grbm_idx_mutex);
5447 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
5448 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
5449 mask = 1;
5450 ao_bitmap = 0;
5451 counter = 0;
5452 gfx_v10_0_select_se_sh(adev, i, j, 0xffffffff);
5453 if (i < 4 && j < 2)
5454 gfx_v10_0_set_user_wgp_inactive_bitmap_per_sh(
5455 adev, disable_masks[i * 2 + j]);
5456 bitmap = gfx_v10_0_get_cu_active_bitmap_per_sh(adev);
5457 cu_info->bitmap[i][j] = bitmap;
5458
5459 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) {
5460 if (bitmap & mask) {
5461 if (counter < adev->gfx.config.max_cu_per_sh)
5462 ao_bitmap |= mask;
5463 counter++;
5464 }
5465 mask <<= 1;
5466 }
5467 active_cu_number += counter;
5468 if (i < 2 && j < 2)
5469 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
5470 cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
5471 }
5472 }
5473 gfx_v10_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
5474 mutex_unlock(&adev->grbm_idx_mutex);
5475
5476 cu_info->number = active_cu_number;
5477 cu_info->ao_cu_mask = ao_cu_mask;
5478 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
5479
5480 return 0;
5481 }
5482
5483 const struct amdgpu_ip_block_version gfx_v10_0_ip_block =
5484 {
5485 .type = AMD_IP_BLOCK_TYPE_GFX,
5486 .major = 10,
5487 .minor = 0,
5488 .rev = 0,
5489 .funcs = &gfx_v10_0_ip_funcs,
5490 };