152 static uint32_t cik_sdma_ring_get_rptr(struct amdgpu_ring *ring)  in cik_sdma_ring_get_rptr()  argument
156 	rptr = ring->adev->wb.wb[ring->rptr_offs];  in cik_sdma_ring_get_rptr()
168 static uint32_t cik_sdma_ring_get_wptr(struct amdgpu_ring *ring)  in cik_sdma_ring_get_wptr()  argument
170 	struct amdgpu_device *adev = ring->adev;  in cik_sdma_ring_get_wptr()
171 	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;  in cik_sdma_ring_get_wptr()
183 static void cik_sdma_ring_set_wptr(struct amdgpu_ring *ring)  in cik_sdma_ring_set_wptr()  argument
185 	struct amdgpu_device *adev = ring->adev;  in cik_sdma_ring_set_wptr()
186 	u32 me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;  in cik_sdma_ring_set_wptr()
188 	WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], (ring->wptr << 2) & 0x3fffc);  in cik_sdma_ring_set_wptr()
191 static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)  in cik_sdma_ring_insert_nop()  argument
193 	struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ring);  in cik_sdma_ring_insert_nop()
198 			amdgpu_ring_write(ring, ring->nop |  in cik_sdma_ring_insert_nop()
201 			amdgpu_ring_write(ring, ring->nop);  in cik_sdma_ring_insert_nop()
212 static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring,  in cik_sdma_ring_emit_ib()  argument
215 	u32 extra_bits = (ib->vm ? ib->vm->ids[ring->idx].id : 0) & 0xf;  in cik_sdma_ring_emit_ib()
216 	u32 next_rptr = ring->wptr + 5;  in cik_sdma_ring_emit_ib()
222 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));  in cik_sdma_ring_emit_ib()
223 	amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);  in cik_sdma_ring_emit_ib()
224 	amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);  in cik_sdma_ring_emit_ib()
225 	amdgpu_ring_write(ring, 1); /* number of DWs to follow */  in cik_sdma_ring_emit_ib()
226 	amdgpu_ring_write(ring, next_rptr);  in cik_sdma_ring_emit_ib()
229 	cik_sdma_ring_insert_nop(ring, (12 - (ring->wptr & 7)) % 8);  in cik_sdma_ring_emit_ib()
231 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits));  in cik_sdma_ring_emit_ib()
232 	amdgpu_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */  in cik_sdma_ring_emit_ib()
233 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff);  in cik_sdma_ring_emit_ib()
234 	amdgpu_ring_write(ring, ib->length_dw);  in cik_sdma_ring_emit_ib()
245 static void cik_sdma_ring_emit_hdp_flush(struct amdgpu_ring *ring)  in cik_sdma_ring_emit_hdp_flush()  argument
251 	if (ring == &ring->adev->sdma.instance[0].ring)  in cik_sdma_ring_emit_hdp_flush()
256 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));  in cik_sdma_ring_emit_hdp_flush()
257 	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE << 2);  in cik_sdma_ring_emit_hdp_flush()
258 	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ << 2);  in cik_sdma_ring_emit_hdp_flush()
259 	amdgpu_ring_write(ring, ref_and_mask); /* reference */  in cik_sdma_ring_emit_hdp_flush()
260 	amdgpu_ring_write(ring, ref_and_mask); /* mask */  in cik_sdma_ring_emit_hdp_flush()
261 	amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */  in cik_sdma_ring_emit_hdp_flush()
274 static void cik_sdma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,  in cik_sdma_ring_emit_fence()  argument
279 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));  in cik_sdma_ring_emit_fence()
280 	amdgpu_ring_write(ring, lower_32_bits(addr));  in cik_sdma_ring_emit_fence()
281 	amdgpu_ring_write(ring, upper_32_bits(addr));  in cik_sdma_ring_emit_fence()
282 	amdgpu_ring_write(ring, lower_32_bits(seq));  in cik_sdma_ring_emit_fence()
287 		amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));  in cik_sdma_ring_emit_fence()
288 		amdgpu_ring_write(ring, lower_32_bits(addr));  in cik_sdma_ring_emit_fence()
289 		amdgpu_ring_write(ring, upper_32_bits(addr));  in cik_sdma_ring_emit_fence()
290 		amdgpu_ring_write(ring, upper_32_bits(seq));  in cik_sdma_ring_emit_fence()
294 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));  in cik_sdma_ring_emit_fence()
307 static bool cik_sdma_ring_emit_semaphore(struct amdgpu_ring *ring,  in cik_sdma_ring_emit_semaphore()  argument
314 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SEMAPHORE, 0, extra_bits));  in cik_sdma_ring_emit_semaphore()
315 	amdgpu_ring_write(ring, addr & 0xfffffff8);  in cik_sdma_ring_emit_semaphore()
316 	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);  in cik_sdma_ring_emit_semaphore()
330 	struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring;  in cik_sdma_gfx_stop()
331 	struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring;  in cik_sdma_gfx_stop()
399 	struct amdgpu_ring *ring;  in cik_sdma_gfx_resume()  local
406 		ring = &adev->sdma.instance[i].ring;  in cik_sdma_gfx_resume()
407 		wb_offset = (ring->rptr_offs * 4);  in cik_sdma_gfx_resume()
424 		rb_bufsz = order_base_2(ring->ring_size / 4);  in cik_sdma_gfx_resume()
444 		WREG32(mmSDMA0_GFX_RB_BASE + sdma_offsets[i], ring->gpu_addr >> 8);  in cik_sdma_gfx_resume()
445 		WREG32(mmSDMA0_GFX_RB_BASE_HI + sdma_offsets[i], ring->gpu_addr >> 40);  in cik_sdma_gfx_resume()
447 		ring->wptr = 0;  in cik_sdma_gfx_resume()
448 		WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[i], ring->wptr << 2);  in cik_sdma_gfx_resume()
461 		ring->ready = true;  in cik_sdma_gfx_resume()
463 		r = amdgpu_ring_test_ring(ring);  in cik_sdma_gfx_resume()
465 			ring->ready = false;  in cik_sdma_gfx_resume()
469 		if (adev->mman.buffer_funcs_ring == ring)  in cik_sdma_gfx_resume()
568 static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)  in cik_sdma_ring_test_ring()  argument
570 	struct amdgpu_device *adev = ring->adev;  in cik_sdma_ring_test_ring()
587 	r = amdgpu_ring_lock(ring, 5);  in cik_sdma_ring_test_ring()
589 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);  in cik_sdma_ring_test_ring()
593 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));  in cik_sdma_ring_test_ring()
594 	amdgpu_ring_write(ring, lower_32_bits(gpu_addr));  in cik_sdma_ring_test_ring()
595 	amdgpu_ring_write(ring, upper_32_bits(gpu_addr));  in cik_sdma_ring_test_ring()
596 	amdgpu_ring_write(ring, 1); /* number of DWs to follow */  in cik_sdma_ring_test_ring()
597 	amdgpu_ring_write(ring, 0xDEADBEEF);  in cik_sdma_ring_test_ring()
598 	amdgpu_ring_unlock_commit(ring);  in cik_sdma_ring_test_ring()
608 		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);  in cik_sdma_ring_test_ring()
611 			  ring->idx, tmp);  in cik_sdma_ring_test_ring()
627 static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring)  in cik_sdma_ring_test_ib()  argument
629 	struct amdgpu_device *adev = ring->adev;  in cik_sdma_ring_test_ib()
648 	r = amdgpu_ib_get(ring, NULL, 256, &ib);  in cik_sdma_ring_test_ib()
660 	r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL,  in cik_sdma_ring_test_ib()
679 			 ring->idx, i);  in cik_sdma_ring_test_ib()
761 				value = amdgpu_vm_map_gart(ib->ring->adev, addr);  in cik_sdma_vm_write_pte()
832 	struct amdgpu_sdma_instance *sdma = amdgpu_get_sdma_instance(ib->ring);  in cik_sdma_vm_pad_ib()
856 static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,  in cik_sdma_ring_emit_vm_flush()  argument
862 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_sdma_ring_emit_vm_flush()
864 		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));  in cik_sdma_ring_emit_vm_flush()
866 		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8));  in cik_sdma_ring_emit_vm_flush()
868 	amdgpu_ring_write(ring, pd_addr >> 12);  in cik_sdma_ring_emit_vm_flush()
871 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));  in cik_sdma_ring_emit_vm_flush()
872 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);  in cik_sdma_ring_emit_vm_flush()
873 	amdgpu_ring_write(ring, 1 << vm_id);  in cik_sdma_ring_emit_vm_flush()
875 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));  in cik_sdma_ring_emit_vm_flush()
876 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);  in cik_sdma_ring_emit_vm_flush()
877 	amdgpu_ring_write(ring, 0);  in cik_sdma_ring_emit_vm_flush()
878 	amdgpu_ring_write(ring, 0); /* reference */  in cik_sdma_ring_emit_vm_flush()
879 	amdgpu_ring_write(ring, 0); /* mask */  in cik_sdma_ring_emit_vm_flush()
880 	amdgpu_ring_write(ring, (0xfff << 16) | 10); /* retry count, poll interval */  in cik_sdma_ring_emit_vm_flush()
948 	struct amdgpu_ring *ring;  in cik_sdma_sw_init()  local
974 		ring = &adev->sdma.instance[i].ring;  in cik_sdma_sw_init()
975 		ring->ring_obj = NULL;  in cik_sdma_sw_init()
976 		sprintf(ring->name, "sdma%d", i);  in cik_sdma_sw_init()
977 		r = amdgpu_ring_init(adev, ring, 256 * 1024,  in cik_sdma_sw_init()
996 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);  in cik_sdma_sw_fini()
1218 			amdgpu_fence_process(&adev->sdma.instance[0].ring);  in cik_sdma_process_trap_irq()
1231 			amdgpu_fence_process(&adev->sdma.instance[1].ring);  in cik_sdma_process_trap_irq()
1313 		adev->sdma.instance[i].ring.funcs = &cik_sdma_ring_funcs;  in cik_sdma_set_ring_funcs()
1394 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;  in cik_sdma_set_buffer_funcs()
1409 		adev->vm_manager.vm_pte_funcs_ring = &adev->sdma.instance[0].ring;  in cik_sdma_set_vm_pte_funcs()