51 static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev);
52 static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev);
53 static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev);
54 static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev);
67 u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev);
95 static int cik_sdma_init_microcode(struct amdgpu_device *adev)  in cik_sdma_init_microcode()  argument
103 	switch (adev->asic_type) {  in cik_sdma_init_microcode()
122 	for (i = 0; i < adev->sdma.num_instances; i++) {  in cik_sdma_init_microcode()
127 		err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev);  in cik_sdma_init_microcode()
130 		err = amdgpu_ucode_validate(adev->sdma.instance[i].fw);  in cik_sdma_init_microcode()
137 		for (i = 0; i < adev->sdma.num_instances; i++) {  in cik_sdma_init_microcode()
138 			release_firmware(adev->sdma.instance[i].fw);  in cik_sdma_init_microcode()
139 			adev->sdma.instance[i].fw = NULL;  in cik_sdma_init_microcode()
156 	rptr = ring->adev->wb.wb[ring->rptr_offs];  in cik_sdma_ring_get_rptr()
170 	struct amdgpu_device *adev = ring->adev;  in cik_sdma_ring_get_wptr()  local
171 	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;  in cik_sdma_ring_get_wptr()
185 	struct amdgpu_device *adev = ring->adev;  in cik_sdma_ring_set_wptr()  local
186 	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;  in cik_sdma_ring_set_wptr()
251 	if (ring == &ring->adev->sdma.instance[0].ring)  in cik_sdma_ring_emit_hdp_flush()
328 static void cik_sdma_gfx_stop(struct amdgpu_device *adev)  in cik_sdma_gfx_stop()  argument
330 	struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;  in cik_sdma_gfx_stop()
331 	struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;  in cik_sdma_gfx_stop()
335 	if ((adev->mman.buffer_funcs_ring == sdma0) ||  in cik_sdma_gfx_stop()
336 	    (adev->mman.buffer_funcs_ring == sdma1))  in cik_sdma_gfx_stop()
337 		amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);  in cik_sdma_gfx_stop()
339 	for (i = 0; i < adev->sdma.num_instances; i++) {  in cik_sdma_gfx_stop()
356 static void cik_sdma_rlc_stop(struct amdgpu_device *adev)  in cik_sdma_rlc_stop()  argument
369 static void cik_sdma_enable(struct amdgpu_device *adev, bool enable)  in cik_sdma_enable()  argument
375 		cik_sdma_gfx_stop(adev);  in cik_sdma_enable()
376 		cik_sdma_rlc_stop(adev);  in cik_sdma_enable()
379 	for (i = 0; i < adev->sdma.num_instances; i++) {  in cik_sdma_enable()
397 static int cik_sdma_gfx_resume(struct amdgpu_device *adev)  in cik_sdma_gfx_resume()  argument
405 	for (i = 0; i < adev->sdma.num_instances; i++) {  in cik_sdma_gfx_resume()
406 		ring = &adev->sdma.instance[i].ring;  in cik_sdma_gfx_resume()
409 		mutex_lock(&adev->srbm_mutex);  in cik_sdma_gfx_resume()
411 			cik_srbm_select(adev, 0, 0, 0, j);  in cik_sdma_gfx_resume()
417 		cik_srbm_select(adev, 0, 0, 0, 0);  in cik_sdma_gfx_resume()
418 		mutex_unlock(&adev->srbm_mutex);  in cik_sdma_gfx_resume()
438 		       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);  in cik_sdma_gfx_resume()
440 		       ((adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));  in cik_sdma_gfx_resume()
469 		if (adev->mman.buffer_funcs_ring == ring)  in cik_sdma_gfx_resume()
470 			amdgpu_ttm_set_active_vram_size(adev, adev->mc.real_vram_size);  in cik_sdma_gfx_resume()
484 static int cik_sdma_rlc_resume(struct amdgpu_device *adev)  in cik_sdma_rlc_resume()  argument
498 static int cik_sdma_load_microcode(struct amdgpu_device *adev)  in cik_sdma_load_microcode()  argument
506 	cik_sdma_enable(adev, false);  in cik_sdma_load_microcode()
508 	for (i = 0; i < adev->sdma.num_instances; i++) {  in cik_sdma_load_microcode()
509 		if (!adev->sdma.instance[i].fw)  in cik_sdma_load_microcode()
511 		hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;  in cik_sdma_load_microcode()
514 		adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version);  in cik_sdma_load_microcode()
515 		adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);  in cik_sdma_load_microcode()
516 		if (adev->sdma.instance[i].feature_version >= 20)  in cik_sdma_load_microcode()
517 			adev->sdma.instance[i].burst_nop = true;  in cik_sdma_load_microcode()
519 			(adev->sdma.instance[i].fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_sdma_load_microcode()
523 		WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], adev->sdma.instance[i].fw_version);  in cik_sdma_load_microcode()
537 static int cik_sdma_start(struct amdgpu_device *adev)  in cik_sdma_start()  argument
541 	r = cik_sdma_load_microcode(adev);  in cik_sdma_start()
546 	cik_sdma_enable(adev, true);  in cik_sdma_start()
549 	r = cik_sdma_gfx_resume(adev);  in cik_sdma_start()
552 	r = cik_sdma_rlc_resume(adev);  in cik_sdma_start()
570 	struct amdgpu_device *adev = ring->adev;  in cik_sdma_ring_test_ring()  local
577 	r = amdgpu_wb_get(adev, &index);  in cik_sdma_ring_test_ring()
579 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);  in cik_sdma_ring_test_ring()
583 	gpu_addr = adev->wb.gpu_addr + (index * 4);  in cik_sdma_ring_test_ring()
585 	adev->wb.wb[index] = cpu_to_le32(tmp);  in cik_sdma_ring_test_ring()
590 		amdgpu_wb_free(adev, index);  in cik_sdma_ring_test_ring()
600 	for (i = 0; i < adev->usec_timeout; i++) {  in cik_sdma_ring_test_ring()
601 		tmp = le32_to_cpu(adev->wb.wb[index]);  in cik_sdma_ring_test_ring()
607 	if (i < adev->usec_timeout) {  in cik_sdma_ring_test_ring()
614 	amdgpu_wb_free(adev, index);  in cik_sdma_ring_test_ring()
629 	struct amdgpu_device *adev = ring->adev;  in cik_sdma_ring_test_ib()  local
638 	r = amdgpu_wb_get(adev, &index);  in cik_sdma_ring_test_ib()
640 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);  in cik_sdma_ring_test_ib()
644 	gpu_addr = adev->wb.gpu_addr + (index * 4);  in cik_sdma_ring_test_ib()
646 	adev->wb.wb[index] = cpu_to_le32(tmp);  in cik_sdma_ring_test_ib()
660 	r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL,  in cik_sdma_ring_test_ib()
671 	for (i = 0; i < adev->usec_timeout; i++) {  in cik_sdma_ring_test_ib()
672 		tmp = le32_to_cpu(adev->wb.wb[index]);  in cik_sdma_ring_test_ib()
677 	if (i < adev->usec_timeout) {  in cik_sdma_ring_test_ib()
688 	amdgpu_ib_free(adev, &ib);  in cik_sdma_ring_test_ib()
690 	amdgpu_wb_free(adev, index);  in cik_sdma_ring_test_ib()
761 				value = amdgpu_vm_map_gart(ib->ring->adev, addr);  in cik_sdma_vm_write_pte()
883 static void cik_enable_sdma_mgcg(struct amdgpu_device *adev,  in cik_enable_sdma_mgcg()  argument
888 	if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_SDMA_MGCG)) {  in cik_enable_sdma_mgcg()
904 static void cik_enable_sdma_mgls(struct amdgpu_device *adev,  in cik_enable_sdma_mgls()  argument
909 	if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_SDMA_LS)) {  in cik_enable_sdma_mgls()
934 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_early_init()  local
936 	adev->sdma.num_instances = SDMA_MAX_INSTANCE;  in cik_sdma_early_init()
938 	cik_sdma_set_ring_funcs(adev);  in cik_sdma_early_init()
939 	cik_sdma_set_irq_funcs(adev);  in cik_sdma_early_init()
940 	cik_sdma_set_buffer_funcs(adev);  in cik_sdma_early_init()
941 	cik_sdma_set_vm_pte_funcs(adev);  in cik_sdma_early_init()
949 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_sw_init()  local
952 	r = cik_sdma_init_microcode(adev);  in cik_sdma_sw_init()
959 	r = amdgpu_irq_add_id(adev, 224, &adev->sdma.trap_irq);  in cik_sdma_sw_init()
964 	r = amdgpu_irq_add_id(adev, 241, &adev->sdma.illegal_inst_irq);  in cik_sdma_sw_init()
969 	r = amdgpu_irq_add_id(adev, 247, &adev->sdma.illegal_inst_irq);  in cik_sdma_sw_init()
973 	for (i = 0; i < adev->sdma.num_instances; i++) {  in cik_sdma_sw_init()
974 		ring = &adev->sdma.instance[i].ring;  in cik_sdma_sw_init()
977 		r = amdgpu_ring_init(adev, ring, 256 * 1024,  in cik_sdma_sw_init()
979 				     &adev->sdma.trap_irq,  in cik_sdma_sw_init()
992 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_sw_fini()  local
995 	for (i = 0; i < adev->sdma.num_instances; i++)  in cik_sdma_sw_fini()
996 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);  in cik_sdma_sw_fini()
1004 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_hw_init()  local
1006 	r = cik_sdma_start(adev);  in cik_sdma_hw_init()
1015 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_hw_fini()  local
1017 	cik_sdma_enable(adev, false);  in cik_sdma_hw_fini()
1024 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_suspend()  local
1026 	return cik_sdma_hw_fini(adev);  in cik_sdma_suspend()
1031 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_resume()  local
1033 	return cik_sdma_hw_init(adev);  in cik_sdma_resume()
1038 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_is_idle()  local
1052 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_wait_for_idle()  local
1054 	for (i = 0; i < adev->usec_timeout; i++) {  in cik_sdma_wait_for_idle()
1068 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_print_status()  local
1070 	dev_info(adev->dev, "CIK SDMA registers\n");  in cik_sdma_print_status()
1071 	dev_info(adev->dev, "  SRBM_STATUS2=0x%08X\n",  in cik_sdma_print_status()
1073 	for (i = 0; i < adev->sdma.num_instances; i++) {  in cik_sdma_print_status()
1074 		dev_info(adev->dev, "  SDMA%d_STATUS_REG=0x%08X\n",  in cik_sdma_print_status()
1076 		dev_info(adev->dev, "  SDMA%d_ME_CNTL=0x%08X\n",  in cik_sdma_print_status()
1078 		dev_info(adev->dev, "  SDMA%d_CNTL=0x%08X\n",  in cik_sdma_print_status()
1080 		dev_info(adev->dev, "  SDMA%d_SEM_INCOMPLETE_TIMER_CNTL=0x%08X\n",  in cik_sdma_print_status()
1082 		dev_info(adev->dev, "  SDMA%d_SEM_WAIT_FAIL_TIMER_CNTL=0x%08X\n",  in cik_sdma_print_status()
1084 		dev_info(adev->dev, "  SDMA%d_GFX_IB_CNTL=0x%08X\n",  in cik_sdma_print_status()
1086 		dev_info(adev->dev, "  SDMA%d_GFX_RB_CNTL=0x%08X\n",  in cik_sdma_print_status()
1088 		dev_info(adev->dev, "  SDMA%d_GFX_RB_RPTR=0x%08X\n",  in cik_sdma_print_status()
1090 		dev_info(adev->dev, "  SDMA%d_GFX_RB_WPTR=0x%08X\n",  in cik_sdma_print_status()
1092 		dev_info(adev->dev, "  SDMA%d_GFX_RB_RPTR_ADDR_HI=0x%08X\n",  in cik_sdma_print_status()
1094 		dev_info(adev->dev, "  SDMA%d_GFX_RB_RPTR_ADDR_LO=0x%08X\n",  in cik_sdma_print_status()
1096 		dev_info(adev->dev, "  SDMA%d_GFX_RB_BASE=0x%08X\n",  in cik_sdma_print_status()
1098 		dev_info(adev->dev, "  SDMA%d_GFX_RB_BASE_HI=0x%08X\n",  in cik_sdma_print_status()
1100 		mutex_lock(&adev->srbm_mutex);  in cik_sdma_print_status()
1102 			cik_srbm_select(adev, 0, 0, 0, j);  in cik_sdma_print_status()
1103 			dev_info(adev->dev, "  VM %d:\n", j);  in cik_sdma_print_status()
1104 			dev_info(adev->dev, "  SDMA0_GFX_VIRTUAL_ADDR=0x%08X\n",  in cik_sdma_print_status()
1106 			dev_info(adev->dev, "  SDMA0_GFX_APE1_CNTL=0x%08X\n",  in cik_sdma_print_status()
1109 		cik_srbm_select(adev, 0, 0, 0, 0);  in cik_sdma_print_status()
1110 		mutex_unlock(&adev->srbm_mutex);  in cik_sdma_print_status()
1117 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_soft_reset()  local
1136 		cik_sdma_print_status((void *)adev);  in cik_sdma_soft_reset()
1140 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in cik_sdma_soft_reset()
1153 		cik_sdma_print_status((void *)adev);  in cik_sdma_soft_reset()
1159 static int cik_sdma_set_trap_irq_state(struct amdgpu_device *adev,  in cik_sdma_set_trap_irq_state()  argument
1205 static int cik_sdma_process_trap_irq(struct amdgpu_device *adev,  in cik_sdma_process_trap_irq()  argument
1218 			amdgpu_fence_process(&adev->sdma.instance[0].ring);  in cik_sdma_process_trap_irq()
1231 			amdgpu_fence_process(&adev->sdma.instance[1].ring);  in cik_sdma_process_trap_irq()
1246 static int cik_sdma_process_illegal_inst_irq(struct amdgpu_device *adev,  in cik_sdma_process_illegal_inst_irq()  argument
1251 	schedule_work(&adev->reset_work);  in cik_sdma_process_illegal_inst_irq()
1259 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;  in cik_sdma_set_clockgating_state()  local
1264 	cik_enable_sdma_mgcg(adev, gate);  in cik_sdma_set_clockgating_state()
1265 	cik_enable_sdma_mgls(adev, gate);  in cik_sdma_set_clockgating_state()
1308 static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev)  in cik_sdma_set_ring_funcs()  argument
1312 	for (i = 0; i < adev->sdma.num_instances; i++)  in cik_sdma_set_ring_funcs()
1313 		adev->sdma.instance[i].ring.funcs = &cik_sdma_ring_funcs;  in cik_sdma_set_ring_funcs()
1325 static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev)  in cik_sdma_set_irq_funcs()  argument
1327 	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;  in cik_sdma_set_irq_funcs()
1328 	adev->sdma.trap_irq.funcs = &cik_sdma_trap_irq_funcs;  in cik_sdma_set_irq_funcs()
1329 	adev->sdma.illegal_inst_irq.funcs = &cik_sdma_illegal_inst_irq_funcs;  in cik_sdma_set_irq_funcs()
1390 static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)  in cik_sdma_set_buffer_funcs()  argument
1392 	if (adev->mman.buffer_funcs == NULL) {  in cik_sdma_set_buffer_funcs()
1393 		adev->mman.buffer_funcs = &cik_sdma_buffer_funcs;  in cik_sdma_set_buffer_funcs()
1394 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;  in cik_sdma_set_buffer_funcs()
1405 static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev)  in cik_sdma_set_vm_pte_funcs()  argument
1407 	if (adev->vm_manager.vm_pte_funcs == NULL) {  in cik_sdma_set_vm_pte_funcs()
1408 		adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs;  in cik_sdma_set_vm_pte_funcs()
1409 		adev->vm_manager.vm_pte_funcs_ring = &adev->sdma.instance[0].ring;  in cik_sdma_set_vm_pte_funcs()
1410 		adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true;  in cik_sdma_set_vm_pte_funcs()