36 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev);
63 uint32_t cik_sdma_get_rptr(struct radeon_device *rdev,  in cik_sdma_get_rptr()  argument
68 	if (rdev->wb.enabled) {  in cik_sdma_get_rptr()
69 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in cik_sdma_get_rptr()
90 uint32_t cik_sdma_get_wptr(struct radeon_device *rdev,  in cik_sdma_get_wptr()  argument
111 void cik_sdma_set_wptr(struct radeon_device *rdev,  in cik_sdma_set_wptr()  argument
133 void cik_sdma_ring_ib_execute(struct radeon_device *rdev,  in cik_sdma_ring_ib_execute()  argument
136 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in cik_sdma_ring_ib_execute()
139 	if (rdev->wb.enabled) {  in cik_sdma_ring_ib_execute()
169 static void cik_sdma_hdp_flush_ring_emit(struct radeon_device *rdev,  in cik_sdma_hdp_flush_ring_emit()  argument
172 	struct radeon_ring *ring = &rdev->ring[ridx];  in cik_sdma_hdp_flush_ring_emit()
200 void cik_sdma_fence_ring_emit(struct radeon_device *rdev,  in cik_sdma_fence_ring_emit()  argument
203 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cik_sdma_fence_ring_emit()
204 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cik_sdma_fence_ring_emit()
214 	cik_sdma_hdp_flush_ring_emit(rdev, fence->ring);  in cik_sdma_fence_ring_emit()
228 bool cik_sdma_semaphore_ring_emit(struct radeon_device *rdev,  in cik_sdma_semaphore_ring_emit()  argument
250 static void cik_sdma_gfx_stop(struct radeon_device *rdev)  in cik_sdma_gfx_stop()  argument
255 	if ((rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) ||  in cik_sdma_gfx_stop()
256 	    (rdev->asic->copy.copy_ring_index == CAYMAN_RING_TYPE_DMA1_INDEX))  in cik_sdma_gfx_stop()
257 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);  in cik_sdma_gfx_stop()
269 	rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;  in cik_sdma_gfx_stop()
270 	rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;  in cik_sdma_gfx_stop()
291 static void cik_sdma_rlc_stop(struct radeon_device *rdev)  in cik_sdma_rlc_stop()  argument
304 static void cik_sdma_ctx_switch_enable(struct radeon_device *rdev, bool enable)  in cik_sdma_ctx_switch_enable()  argument
331 void cik_sdma_enable(struct radeon_device *rdev, bool enable)  in cik_sdma_enable()  argument
337 		cik_sdma_gfx_stop(rdev);  in cik_sdma_enable()
338 		cik_sdma_rlc_stop(rdev);  in cik_sdma_enable()
354 	cik_sdma_ctx_switch_enable(rdev, enable);  in cik_sdma_enable()
365 static int cik_sdma_gfx_resume(struct radeon_device *rdev)  in cik_sdma_gfx_resume()  argument
375 			ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cik_sdma_gfx_resume()
379 			ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cik_sdma_gfx_resume()
401 		       upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);  in cik_sdma_gfx_resume()
403 		       ((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));  in cik_sdma_gfx_resume()
405 		if (rdev->wb.enabled)  in cik_sdma_gfx_resume()
426 		r = radeon_ring_test(rdev, ring->idx, ring);  in cik_sdma_gfx_resume()
433 	if ((rdev->asic->copy.copy_ring_index == R600_RING_TYPE_DMA_INDEX) ||  in cik_sdma_gfx_resume()
434 	    (rdev->asic->copy.copy_ring_index == CAYMAN_RING_TYPE_DMA1_INDEX))  in cik_sdma_gfx_resume()
435 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);  in cik_sdma_gfx_resume()
448 static int cik_sdma_rlc_resume(struct radeon_device *rdev)  in cik_sdma_rlc_resume()  argument
462 static int cik_sdma_load_microcode(struct radeon_device *rdev)  in cik_sdma_load_microcode()  argument
466 	if (!rdev->sdma_fw)  in cik_sdma_load_microcode()
470 	cik_sdma_enable(rdev, false);  in cik_sdma_load_microcode()
472 	if (rdev->new_fw) {  in cik_sdma_load_microcode()
474 			(const struct sdma_firmware_header_v1_0 *)rdev->sdma_fw->data;  in cik_sdma_load_microcode()
482 			(rdev->sdma_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_sdma_load_microcode()
491 			(rdev->sdma_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_sdma_load_microcode()
501 		fw_data = (const __be32 *)rdev->sdma_fw->data;  in cik_sdma_load_microcode()
508 		fw_data = (const __be32 *)rdev->sdma_fw->data;  in cik_sdma_load_microcode()
528 int cik_sdma_resume(struct radeon_device *rdev)  in cik_sdma_resume()  argument
532 	r = cik_sdma_load_microcode(rdev);  in cik_sdma_resume()
537 	cik_sdma_enable(rdev, true);  in cik_sdma_resume()
540 	r = cik_sdma_gfx_resume(rdev);  in cik_sdma_resume()
543 	r = cik_sdma_rlc_resume(rdev);  in cik_sdma_resume()
557 void cik_sdma_fini(struct radeon_device *rdev)  in cik_sdma_fini()  argument
560 	cik_sdma_enable(rdev, false);  in cik_sdma_fini()
561 	radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);  in cik_sdma_fini()
562 	radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);  in cik_sdma_fini()
579 struct radeon_fence *cik_copy_dma(struct radeon_device *rdev,  in cik_copy_dma()  argument
586 	int ring_index = rdev->asic->copy.dma_ring_index;  in cik_copy_dma()
587 	struct radeon_ring *ring = &rdev->ring[ring_index];  in cik_copy_dma()
596 	r = radeon_ring_lock(rdev, ring, num_loops * 7 + 14);  in cik_copy_dma()
599 		radeon_sync_free(rdev, &sync, NULL);  in cik_copy_dma()
603 	radeon_sync_resv(rdev, &sync, resv, false);  in cik_copy_dma()
604 	radeon_sync_rings(rdev, &sync, ring->idx);  in cik_copy_dma()
622 	r = radeon_fence_emit(rdev, &fence, ring->idx);  in cik_copy_dma()
624 		radeon_ring_unlock_undo(rdev, ring);  in cik_copy_dma()
625 		radeon_sync_free(rdev, &sync, NULL);  in cik_copy_dma()
629 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_copy_dma()
630 	radeon_sync_free(rdev, &sync, fence);  in cik_copy_dma()
645 int cik_sdma_ring_test(struct radeon_device *rdev,  in cik_sdma_ring_test()  argument
659 	gpu_addr = rdev->wb.gpu_addr + index;  in cik_sdma_ring_test()
662 	rdev->wb.wb[index/4] = cpu_to_le32(tmp);  in cik_sdma_ring_test()
664 	r = radeon_ring_lock(rdev, ring, 5);  in cik_sdma_ring_test()
674 	radeon_ring_unlock_commit(rdev, ring, false);  in cik_sdma_ring_test()
676 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_sdma_ring_test()
677 		tmp = le32_to_cpu(rdev->wb.wb[index/4]);  in cik_sdma_ring_test()
683 	if (i < rdev->usec_timeout) {  in cik_sdma_ring_test()
702 int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_sdma_ib_test()  argument
716 	gpu_addr = rdev->wb.gpu_addr + index;  in cik_sdma_ib_test()
719 	rdev->wb.wb[index/4] = cpu_to_le32(tmp);  in cik_sdma_ib_test()
721 	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);  in cik_sdma_ib_test()
734 	r = radeon_ib_schedule(rdev, &ib, NULL, false);  in cik_sdma_ib_test()
736 		radeon_ib_free(rdev, &ib);  in cik_sdma_ib_test()
745 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_sdma_ib_test()
746 		tmp = le32_to_cpu(rdev->wb.wb[index/4]);  in cik_sdma_ib_test()
751 	if (i < rdev->usec_timeout) {  in cik_sdma_ib_test()
757 	radeon_ib_free(rdev, &ib);  in cik_sdma_ib_test()
770 bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in cik_sdma_is_lockup()  argument
772 	u32 reset_mask = cik_gpu_check_soft_reset(rdev);  in cik_sdma_is_lockup()
781 		radeon_ring_lockup_update(rdev, ring);  in cik_sdma_is_lockup()
784 	return radeon_ring_test_lockup(rdev, ring);  in cik_sdma_is_lockup()
798 void cik_sdma_vm_copy_pages(struct radeon_device *rdev,  in cik_sdma_vm_copy_pages()  argument
836 void cik_sdma_vm_write_pages(struct radeon_device *rdev,  in cik_sdma_vm_write_pages()  argument
858 				value = radeon_vm_map_gart(rdev, addr);  in cik_sdma_vm_write_pages()
885 void cik_sdma_vm_set_pages(struct radeon_device *rdev,  in cik_sdma_vm_set_pages()  argument
942 void cik_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,  in cik_dma_vm_flush()  argument
982 	cik_sdma_hdp_flush_ring_emit(rdev, ring->idx);  in cik_dma_vm_flush()