Lines Matching refs:ring
620 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring) in gfx_v8_0_ring_test_ring() argument
622 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_test_ring()
634 r = amdgpu_ring_lock(ring, 3); in gfx_v8_0_ring_test_ring()
637 ring->idx, r); in gfx_v8_0_ring_test_ring()
641 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); in gfx_v8_0_ring_test_ring()
642 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); in gfx_v8_0_ring_test_ring()
643 amdgpu_ring_write(ring, 0xDEADBEEF); in gfx_v8_0_ring_test_ring()
644 amdgpu_ring_unlock_commit(ring); in gfx_v8_0_ring_test_ring()
654 ring->idx, i); in gfx_v8_0_ring_test_ring()
657 ring->idx, scratch, tmp); in gfx_v8_0_ring_test_ring()
664 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) in gfx_v8_0_ring_test_ib() argument
666 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_test_ib()
681 r = amdgpu_ib_get(ring, NULL, 256, &ib); in gfx_v8_0_ring_test_ib()
691 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, in gfx_v8_0_ring_test_ib()
710 ring->idx, i); in gfx_v8_0_ring_test_ib()
1196 struct amdgpu_ring *ring; in gfx_v8_0_sw_init() local
1232 ring = &adev->gfx.gfx_ring[i]; in gfx_v8_0_sw_init()
1233 ring->ring_obj = NULL; in gfx_v8_0_sw_init()
1234 sprintf(ring->name, "gfx"); in gfx_v8_0_sw_init()
1237 ring->use_doorbell = true; in gfx_v8_0_sw_init()
1238 ring->doorbell_index = AMDGPU_DOORBELL_GFX_RING0; in gfx_v8_0_sw_init()
1241 r = amdgpu_ring_init(adev, ring, 1024 * 1024, in gfx_v8_0_sw_init()
1258 ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_sw_init()
1259 ring->ring_obj = NULL; in gfx_v8_0_sw_init()
1260 ring->use_doorbell = true; in gfx_v8_0_sw_init()
1261 ring->doorbell_index = AMDGPU_DOORBELL_MEC_RING0 + i; in gfx_v8_0_sw_init()
1262 ring->me = 1; /* first MEC */ in gfx_v8_0_sw_init()
1263 ring->pipe = i / 8; in gfx_v8_0_sw_init()
1264 ring->queue = i % 8; in gfx_v8_0_sw_init()
1265 sprintf(ring->name, "comp %d.%d.%d", ring->me, ring->pipe, ring->queue); in gfx_v8_0_sw_init()
1266 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe; in gfx_v8_0_sw_init()
1268 r = amdgpu_ring_init(adev, ring, 1024 * 1024, in gfx_v8_0_sw_init()
3215 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; in gfx_v8_0_cp_gfx_start() local
3227 r = amdgpu_ring_lock(ring, gfx_v8_0_get_csb_size(adev) + 4); in gfx_v8_0_cp_gfx_start()
3234 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); in gfx_v8_0_cp_gfx_start()
3235 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); in gfx_v8_0_cp_gfx_start()
3237 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); in gfx_v8_0_cp_gfx_start()
3238 amdgpu_ring_write(ring, 0x80000000); in gfx_v8_0_cp_gfx_start()
3239 amdgpu_ring_write(ring, 0x80000000); in gfx_v8_0_cp_gfx_start()
3244 amdgpu_ring_write(ring, in gfx_v8_0_cp_gfx_start()
3247 amdgpu_ring_write(ring, in gfx_v8_0_cp_gfx_start()
3250 amdgpu_ring_write(ring, ext->extent[i]); in gfx_v8_0_cp_gfx_start()
3255 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); in gfx_v8_0_cp_gfx_start()
3256 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); in gfx_v8_0_cp_gfx_start()
3259 amdgpu_ring_write(ring, 0x16000012); in gfx_v8_0_cp_gfx_start()
3260 amdgpu_ring_write(ring, 0x0000002A); in gfx_v8_0_cp_gfx_start()
3263 amdgpu_ring_write(ring, 0x3a00161a); in gfx_v8_0_cp_gfx_start()
3264 amdgpu_ring_write(ring, 0x0000002e); in gfx_v8_0_cp_gfx_start()
3268 amdgpu_ring_write(ring, 0x00000002); in gfx_v8_0_cp_gfx_start()
3269 amdgpu_ring_write(ring, 0x00000000); in gfx_v8_0_cp_gfx_start()
3272 amdgpu_ring_write(ring, 0x00000000); in gfx_v8_0_cp_gfx_start()
3273 amdgpu_ring_write(ring, 0x00000000); in gfx_v8_0_cp_gfx_start()
3279 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); in gfx_v8_0_cp_gfx_start()
3280 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); in gfx_v8_0_cp_gfx_start()
3282 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); in gfx_v8_0_cp_gfx_start()
3283 amdgpu_ring_write(ring, 0); in gfx_v8_0_cp_gfx_start()
3286 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); in gfx_v8_0_cp_gfx_start()
3287 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); in gfx_v8_0_cp_gfx_start()
3288 amdgpu_ring_write(ring, 0x8000); in gfx_v8_0_cp_gfx_start()
3289 amdgpu_ring_write(ring, 0x8000); in gfx_v8_0_cp_gfx_start()
3291 amdgpu_ring_unlock_commit(ring); in gfx_v8_0_cp_gfx_start()
3298 struct amdgpu_ring *ring; in gfx_v8_0_cp_gfx_resume() local
3311 ring = &adev->gfx.gfx_ring[0]; in gfx_v8_0_cp_gfx_resume()
3312 rb_bufsz = order_base_2(ring->ring_size / 8); in gfx_v8_0_cp_gfx_resume()
3324 ring->wptr = 0; in gfx_v8_0_cp_gfx_resume()
3325 WREG32(mmCP_RB0_WPTR, ring->wptr); in gfx_v8_0_cp_gfx_resume()
3328 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); in gfx_v8_0_cp_gfx_resume()
3335 rb_addr = ring->gpu_addr >> 8; in gfx_v8_0_cp_gfx_resume()
3342 if (ring->use_doorbell) { in gfx_v8_0_cp_gfx_resume()
3344 DOORBELL_OFFSET, ring->doorbell_index); in gfx_v8_0_cp_gfx_resume()
3367 ring->ready = true; in gfx_v8_0_cp_gfx_resume()
3368 r = amdgpu_ring_test_ring(ring); in gfx_v8_0_cp_gfx_resume()
3370 ring->ready = false; in gfx_v8_0_cp_gfx_resume()
3710 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_cp_compute_fini() local
3712 if (ring->mqd_obj) { in gfx_v8_0_cp_compute_fini()
3713 r = amdgpu_bo_reserve(ring->mqd_obj, false); in gfx_v8_0_cp_compute_fini()
3717 amdgpu_bo_unpin(ring->mqd_obj); in gfx_v8_0_cp_compute_fini()
3718 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v8_0_cp_compute_fini()
3720 amdgpu_bo_unref(&ring->mqd_obj); in gfx_v8_0_cp_compute_fini()
3721 ring->mqd_obj = NULL; in gfx_v8_0_cp_compute_fini()
3767 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_cp_compute_resume() local
3769 if (ring->mqd_obj == NULL) { in gfx_v8_0_cp_compute_resume()
3774 NULL, &ring->mqd_obj); in gfx_v8_0_cp_compute_resume()
3781 r = amdgpu_bo_reserve(ring->mqd_obj, false); in gfx_v8_0_cp_compute_resume()
3786 r = amdgpu_bo_pin(ring->mqd_obj, AMDGPU_GEM_DOMAIN_GTT, in gfx_v8_0_cp_compute_resume()
3793 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&buf); in gfx_v8_0_cp_compute_resume()
3813 vi_srbm_select(adev, ring->me, in gfx_v8_0_cp_compute_resume()
3814 ring->pipe, in gfx_v8_0_cp_compute_resume()
3815 ring->queue, 0); in gfx_v8_0_cp_compute_resume()
3866 hqd_gpu_addr = ring->gpu_addr >> 8; in gfx_v8_0_cp_compute_resume()
3875 (order_base_2(ring->ring_size / 4) - 1)); in gfx_v8_0_cp_compute_resume()
3889 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); in gfx_v8_0_cp_compute_resume()
3899 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); in gfx_v8_0_cp_compute_resume()
3918 DOORBELL_OFFSET, ring->doorbell_index); in gfx_v8_0_cp_compute_resume()
3931 ring->wptr = 0; in gfx_v8_0_cp_compute_resume()
3932 mqd->cp_hqd_pq_wptr = ring->wptr; in gfx_v8_0_cp_compute_resume()
3952 amdgpu_bo_kunmap(ring->mqd_obj); in gfx_v8_0_cp_compute_resume()
3953 amdgpu_bo_unreserve(ring->mqd_obj); in gfx_v8_0_cp_compute_resume()
3967 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_cp_compute_resume() local
3969 ring->ready = true; in gfx_v8_0_cp_compute_resume()
3970 r = amdgpu_ring_test_ring(ring); in gfx_v8_0_cp_compute_resume()
3972 ring->ready = false; in gfx_v8_0_cp_compute_resume()
4400 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring, in gfx_v8_0_ring_emit_gds_switch() argument
4416 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); in gfx_v8_0_ring_emit_gds_switch()
4417 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | in gfx_v8_0_ring_emit_gds_switch()
4419 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base); in gfx_v8_0_ring_emit_gds_switch()
4420 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_gds_switch()
4421 amdgpu_ring_write(ring, gds_base); in gfx_v8_0_ring_emit_gds_switch()
4424 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); in gfx_v8_0_ring_emit_gds_switch()
4425 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | in gfx_v8_0_ring_emit_gds_switch()
4427 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size); in gfx_v8_0_ring_emit_gds_switch()
4428 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_gds_switch()
4429 amdgpu_ring_write(ring, gds_size); in gfx_v8_0_ring_emit_gds_switch()
4432 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); in gfx_v8_0_ring_emit_gds_switch()
4433 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | in gfx_v8_0_ring_emit_gds_switch()
4435 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws); in gfx_v8_0_ring_emit_gds_switch()
4436 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_gds_switch()
4437 amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); in gfx_v8_0_ring_emit_gds_switch()
4440 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); in gfx_v8_0_ring_emit_gds_switch()
4441 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | in gfx_v8_0_ring_emit_gds_switch()
4443 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa); in gfx_v8_0_ring_emit_gds_switch()
4444 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_gds_switch()
4445 amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); in gfx_v8_0_ring_emit_gds_switch()
4473 static u32 gfx_v8_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) in gfx_v8_0_ring_get_rptr_gfx() argument
4477 rptr = ring->adev->wb.wb[ring->rptr_offs]; in gfx_v8_0_ring_get_rptr_gfx()
4482 static u32 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) in gfx_v8_0_ring_get_wptr_gfx() argument
4484 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_get_wptr_gfx()
4487 if (ring->use_doorbell) in gfx_v8_0_ring_get_wptr_gfx()
4489 wptr = ring->adev->wb.wb[ring->wptr_offs]; in gfx_v8_0_ring_get_wptr_gfx()
4496 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) in gfx_v8_0_ring_set_wptr_gfx() argument
4498 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_set_wptr_gfx()
4500 if (ring->use_doorbell) { in gfx_v8_0_ring_set_wptr_gfx()
4502 adev->wb.wb[ring->wptr_offs] = ring->wptr; in gfx_v8_0_ring_set_wptr_gfx()
4503 WDOORBELL32(ring->doorbell_index, ring->wptr); in gfx_v8_0_ring_set_wptr_gfx()
4505 WREG32(mmCP_RB0_WPTR, ring->wptr); in gfx_v8_0_ring_set_wptr_gfx()
4510 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) in gfx_v8_0_ring_emit_hdp_flush() argument
4514 if (ring->type == AMDGPU_RING_TYPE_COMPUTE) { in gfx_v8_0_ring_emit_hdp_flush()
4515 switch (ring->me) { in gfx_v8_0_ring_emit_hdp_flush()
4517 ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe; in gfx_v8_0_ring_emit_hdp_flush()
4520 ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe; in gfx_v8_0_ring_emit_hdp_flush()
4531 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); in gfx_v8_0_ring_emit_hdp_flush()
4532 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ in gfx_v8_0_ring_emit_hdp_flush()
4535 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ); in gfx_v8_0_ring_emit_hdp_flush()
4536 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE); in gfx_v8_0_ring_emit_hdp_flush()
4537 amdgpu_ring_write(ring, ref_and_mask); in gfx_v8_0_ring_emit_hdp_flush()
4538 amdgpu_ring_write(ring, ref_and_mask); in gfx_v8_0_ring_emit_hdp_flush()
4539 amdgpu_ring_write(ring, 0x20); /* poll interval */ in gfx_v8_0_ring_emit_hdp_flush()
4542 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, in gfx_v8_0_ring_emit_ib_gfx() argument
4545 bool need_ctx_switch = ring->current_ctx != ib->ctx; in gfx_v8_0_ring_emit_ib_gfx()
4547 u32 next_rptr = ring->wptr + 5; in gfx_v8_0_ring_emit_ib_gfx()
4557 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); in gfx_v8_0_ring_emit_ib_gfx()
4558 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); in gfx_v8_0_ring_emit_ib_gfx()
4559 amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); in gfx_v8_0_ring_emit_ib_gfx()
4560 amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); in gfx_v8_0_ring_emit_ib_gfx()
4561 amdgpu_ring_write(ring, next_rptr); in gfx_v8_0_ring_emit_ib_gfx()
4565 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); in gfx_v8_0_ring_emit_ib_gfx()
4566 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_ib_gfx()
4575 (ib->vm ? (ib->vm->ids[ring->idx].id << 24) : 0); in gfx_v8_0_ring_emit_ib_gfx()
4577 amdgpu_ring_write(ring, header); in gfx_v8_0_ring_emit_ib_gfx()
4578 amdgpu_ring_write(ring, in gfx_v8_0_ring_emit_ib_gfx()
4583 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); in gfx_v8_0_ring_emit_ib_gfx()
4584 amdgpu_ring_write(ring, control); in gfx_v8_0_ring_emit_ib_gfx()
4587 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring, in gfx_v8_0_ring_emit_ib_compute() argument
4591 u32 next_rptr = ring->wptr + 5; in gfx_v8_0_ring_emit_ib_compute()
4596 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); in gfx_v8_0_ring_emit_ib_compute()
4597 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); in gfx_v8_0_ring_emit_ib_compute()
4598 amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); in gfx_v8_0_ring_emit_ib_compute()
4599 amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); in gfx_v8_0_ring_emit_ib_compute()
4600 amdgpu_ring_write(ring, next_rptr); in gfx_v8_0_ring_emit_ib_compute()
4605 (ib->vm ? (ib->vm->ids[ring->idx].id << 24) : 0); in gfx_v8_0_ring_emit_ib_compute()
4607 amdgpu_ring_write(ring, header); in gfx_v8_0_ring_emit_ib_compute()
4608 amdgpu_ring_write(ring, in gfx_v8_0_ring_emit_ib_compute()
4613 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); in gfx_v8_0_ring_emit_ib_compute()
4614 amdgpu_ring_write(ring, control); in gfx_v8_0_ring_emit_ib_compute()
4617 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr, in gfx_v8_0_ring_emit_fence_gfx() argument
4624 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); in gfx_v8_0_ring_emit_fence_gfx()
4625 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | in gfx_v8_0_ring_emit_fence_gfx()
4629 amdgpu_ring_write(ring, addr & 0xfffffffc); in gfx_v8_0_ring_emit_fence_gfx()
4630 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | in gfx_v8_0_ring_emit_fence_gfx()
4632 amdgpu_ring_write(ring, lower_32_bits(seq)); in gfx_v8_0_ring_emit_fence_gfx()
4633 amdgpu_ring_write(ring, upper_32_bits(seq)); in gfx_v8_0_ring_emit_fence_gfx()
4647 static bool gfx_v8_0_ring_emit_semaphore(struct amdgpu_ring *ring, in gfx_v8_0_ring_emit_semaphore() argument
4654 if (ring->adev->asic_type == CHIP_TOPAZ || in gfx_v8_0_ring_emit_semaphore()
4655 ring->adev->asic_type == CHIP_TONGA || in gfx_v8_0_ring_emit_semaphore()
4656 ring->adev->asic_type == CHIP_FIJI) in gfx_v8_0_ring_emit_semaphore()
4660 amdgpu_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 2)); in gfx_v8_0_ring_emit_semaphore()
4661 amdgpu_ring_write(ring, lower_32_bits(addr)); in gfx_v8_0_ring_emit_semaphore()
4662 amdgpu_ring_write(ring, upper_32_bits(addr)); in gfx_v8_0_ring_emit_semaphore()
4663 amdgpu_ring_write(ring, sel); in gfx_v8_0_ring_emit_semaphore()
4666 if (emit_wait && (ring->type == AMDGPU_RING_TYPE_GFX)) { in gfx_v8_0_ring_emit_semaphore()
4668 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); in gfx_v8_0_ring_emit_semaphore()
4669 amdgpu_ring_write(ring, 0x0); in gfx_v8_0_ring_emit_semaphore()
4675 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring, in gfx_v8_0_ring_emit_vm_flush() argument
4678 int usepfp = (ring->type == AMDGPU_RING_TYPE_GFX); in gfx_v8_0_ring_emit_vm_flush()
4679 uint32_t seq = ring->fence_drv.sync_seq[ring->idx]; in gfx_v8_0_ring_emit_vm_flush()
4680 uint64_t addr = ring->fence_drv.gpu_addr; in gfx_v8_0_ring_emit_vm_flush()
4682 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); in gfx_v8_0_ring_emit_vm_flush()
4683 amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */ in gfx_v8_0_ring_emit_vm_flush()
4686 amdgpu_ring_write(ring, addr & 0xfffffffc); in gfx_v8_0_ring_emit_vm_flush()
4687 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); in gfx_v8_0_ring_emit_vm_flush()
4688 amdgpu_ring_write(ring, seq); in gfx_v8_0_ring_emit_vm_flush()
4689 amdgpu_ring_write(ring, 0xffffffff); in gfx_v8_0_ring_emit_vm_flush()
4690 amdgpu_ring_write(ring, 4); /* poll interval */ in gfx_v8_0_ring_emit_vm_flush()
4694 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); in gfx_v8_0_ring_emit_vm_flush()
4695 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_vm_flush()
4696 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); in gfx_v8_0_ring_emit_vm_flush()
4697 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_vm_flush()
4700 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); in gfx_v8_0_ring_emit_vm_flush()
4701 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | in gfx_v8_0_ring_emit_vm_flush()
4705 amdgpu_ring_write(ring, in gfx_v8_0_ring_emit_vm_flush()
4708 amdgpu_ring_write(ring, in gfx_v8_0_ring_emit_vm_flush()
4711 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_vm_flush()
4712 amdgpu_ring_write(ring, pd_addr >> 12); in gfx_v8_0_ring_emit_vm_flush()
4716 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); in gfx_v8_0_ring_emit_vm_flush()
4717 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | in gfx_v8_0_ring_emit_vm_flush()
4719 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST); in gfx_v8_0_ring_emit_vm_flush()
4720 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_vm_flush()
4721 amdgpu_ring_write(ring, 1 << vm_id); in gfx_v8_0_ring_emit_vm_flush()
4724 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); in gfx_v8_0_ring_emit_vm_flush()
4725 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ in gfx_v8_0_ring_emit_vm_flush()
4728 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST); in gfx_v8_0_ring_emit_vm_flush()
4729 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_vm_flush()
4730 amdgpu_ring_write(ring, 0); /* ref */ in gfx_v8_0_ring_emit_vm_flush()
4731 amdgpu_ring_write(ring, 0); /* mask */ in gfx_v8_0_ring_emit_vm_flush()
4732 amdgpu_ring_write(ring, 0x20); /* poll interval */ in gfx_v8_0_ring_emit_vm_flush()
4737 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); in gfx_v8_0_ring_emit_vm_flush()
4738 amdgpu_ring_write(ring, 0x0); in gfx_v8_0_ring_emit_vm_flush()
4739 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); in gfx_v8_0_ring_emit_vm_flush()
4740 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_vm_flush()
4741 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); in gfx_v8_0_ring_emit_vm_flush()
4742 amdgpu_ring_write(ring, 0); in gfx_v8_0_ring_emit_vm_flush()
4746 static u32 gfx_v8_0_ring_get_rptr_compute(struct amdgpu_ring *ring) in gfx_v8_0_ring_get_rptr_compute() argument
4748 return ring->adev->wb.wb[ring->rptr_offs]; in gfx_v8_0_ring_get_rptr_compute()
4751 static u32 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring) in gfx_v8_0_ring_get_wptr_compute() argument
4753 return ring->adev->wb.wb[ring->wptr_offs]; in gfx_v8_0_ring_get_wptr_compute()
4756 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring) in gfx_v8_0_ring_set_wptr_compute() argument
4758 struct amdgpu_device *adev = ring->adev; in gfx_v8_0_ring_set_wptr_compute()
4761 adev->wb.wb[ring->wptr_offs] = ring->wptr; in gfx_v8_0_ring_set_wptr_compute()
4762 WDOORBELL32(ring->doorbell_index, ring->wptr); in gfx_v8_0_ring_set_wptr_compute()
4765 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring, in gfx_v8_0_ring_emit_fence_compute() argument
4773 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); in gfx_v8_0_ring_emit_fence_compute()
4774 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | in gfx_v8_0_ring_emit_fence_compute()
4779 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); in gfx_v8_0_ring_emit_fence_compute()
4780 amdgpu_ring_write(ring, addr & 0xfffffffc); in gfx_v8_0_ring_emit_fence_compute()
4781 amdgpu_ring_write(ring, upper_32_bits(addr)); in gfx_v8_0_ring_emit_fence_compute()
4782 amdgpu_ring_write(ring, lower_32_bits(seq)); in gfx_v8_0_ring_emit_fence_compute()
4783 amdgpu_ring_write(ring, upper_32_bits(seq)); in gfx_v8_0_ring_emit_fence_compute()
4953 struct amdgpu_ring *ring; in gfx_v8_0_eop_irq() local
4967 ring = &adev->gfx.compute_ring[i]; in gfx_v8_0_eop_irq()
4971 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) in gfx_v8_0_eop_irq()
4972 amdgpu_fence_process(ring); in gfx_v8_0_eop_irq()