45 static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
58 				      struct radeon_ring *ring)  in radeon_ring_supports_scratch_reg()  argument
60 	switch (ring->idx) {  in radeon_ring_supports_scratch_reg()
78 void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_free_size()  argument
80 	uint32_t rptr = radeon_ring_get_rptr(rdev, ring);  in radeon_ring_free_size()
83 	ring->ring_free_dw = rptr + (ring->ring_size / 4);  in radeon_ring_free_size()
84 	ring->ring_free_dw -= ring->wptr;  in radeon_ring_free_size()
85 	ring->ring_free_dw &= ring->ptr_mask;  in radeon_ring_free_size()
86 	if (!ring->ring_free_dw) {  in radeon_ring_free_size()
88 		ring->ring_free_dw = ring->ring_size / 4;  in radeon_ring_free_size()
90 		radeon_ring_lockup_update(rdev, ring);  in radeon_ring_free_size()
104 int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)  in radeon_ring_alloc()  argument
109 	if (ndw > (ring->ring_size / 4))  in radeon_ring_alloc()
113 	radeon_ring_free_size(rdev, ring);  in radeon_ring_alloc()
114 	ndw = (ndw + ring->align_mask) & ~ring->align_mask;  in radeon_ring_alloc()
115 	while (ndw > (ring->ring_free_dw - 1)) {  in radeon_ring_alloc()
116 		radeon_ring_free_size(rdev, ring);  in radeon_ring_alloc()
117 		if (ndw < ring->ring_free_dw) {  in radeon_ring_alloc()
120 		r = radeon_fence_wait_next(rdev, ring->idx);  in radeon_ring_alloc()
124 	ring->count_dw = ndw;  in radeon_ring_alloc()
125 	ring->wptr_old = ring->wptr;  in radeon_ring_alloc()
140 int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)  in radeon_ring_lock()  argument
145 	r = radeon_ring_alloc(rdev, ring, ndw);  in radeon_ring_lock()
164 void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_commit()  argument
170 	if (hdp_flush && rdev->asic->ring[ring->idx]->hdp_flush)  in radeon_ring_commit()
171 		rdev->asic->ring[ring->idx]->hdp_flush(rdev, ring);  in radeon_ring_commit()
173 	while (ring->wptr & ring->align_mask) {  in radeon_ring_commit()
174 		radeon_ring_write(ring, ring->nop);  in radeon_ring_commit()
182 	radeon_ring_set_wptr(rdev, ring);  in radeon_ring_commit()
195 void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_unlock_commit()  argument
198 	radeon_ring_commit(rdev, ring, hdp_flush);  in radeon_ring_unlock_commit()
209 void radeon_ring_undo(struct radeon_ring *ring)  in radeon_ring_undo()  argument
211 	ring->wptr = ring->wptr_old;  in radeon_ring_undo()
221 void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_unlock_undo()  argument
223 	radeon_ring_undo(ring);  in radeon_ring_unlock_undo()
235 			       struct radeon_ring *ring)  in radeon_ring_lockup_update()  argument
237 	atomic_set(&ring->last_rptr, radeon_ring_get_rptr(rdev, ring));  in radeon_ring_lockup_update()
238 	atomic64_set(&ring->last_activity, jiffies_64);  in radeon_ring_lockup_update()
247 bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_test_lockup()  argument
249 	uint32_t rptr = radeon_ring_get_rptr(rdev, ring);  in radeon_ring_test_lockup()
250 	uint64_t last = atomic64_read(&ring->last_activity);  in radeon_ring_test_lockup()
253 	if (rptr != atomic_read(&ring->last_rptr)) {  in radeon_ring_test_lockup()
255 		radeon_ring_lockup_update(rdev, ring);  in radeon_ring_test_lockup()
262 			ring->idx, elapsed);  in radeon_ring_test_lockup()
277 unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_backup()  argument
286 	if (ring->ring_obj == NULL) {  in radeon_ring_backup()
292 	if (!radeon_fence_count_emitted(rdev, ring->idx)) {  in radeon_ring_backup()
298 	if (ring->rptr_save_reg)  in radeon_ring_backup()
299 		ptr = RREG32(ring->rptr_save_reg);  in radeon_ring_backup()
301 		ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);  in radeon_ring_backup()
308 	size = ring->wptr + (ring->ring_size / 4);  in radeon_ring_backup()
310 	size &= ring->ptr_mask;  in radeon_ring_backup()
323 		(*data)[i] = ring->ring[ptr++];  in radeon_ring_backup()
324 		ptr &= ring->ptr_mask;  in radeon_ring_backup()
341 int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_restore()  argument
350 	r = radeon_ring_lock(rdev, ring, size);  in radeon_ring_restore()
355 		radeon_ring_write(ring, data[i]);  in radeon_ring_restore()
358 	radeon_ring_unlock_commit(rdev, ring, false);  in radeon_ring_restore()
375 int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,  in radeon_ring_init()  argument
380 	ring->ring_size = ring_size;  in radeon_ring_init()
381 	ring->rptr_offs = rptr_offs;  in radeon_ring_init()
382 	ring->nop = nop;  in radeon_ring_init()
384 	if (ring->ring_obj == NULL) {  in radeon_ring_init()
385 		r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,  in radeon_ring_init()
387 				     NULL, &ring->ring_obj);  in radeon_ring_init()
392 		r = radeon_bo_reserve(ring->ring_obj, false);  in radeon_ring_init()
395 		r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,  in radeon_ring_init()
396 					&ring->gpu_addr);  in radeon_ring_init()
398 			radeon_bo_unreserve(ring->ring_obj);  in radeon_ring_init()
402 		r = radeon_bo_kmap(ring->ring_obj,  in radeon_ring_init()
403 				       (void **)&ring->ring);  in radeon_ring_init()
404 		radeon_bo_unreserve(ring->ring_obj);  in radeon_ring_init()
410 	ring->ptr_mask = (ring->ring_size / 4) - 1;  in radeon_ring_init()
411 	ring->ring_free_dw = ring->ring_size / 4;  in radeon_ring_init()
413 		u32 index = RADEON_WB_RING0_NEXT_RPTR + (ring->idx * 4);  in radeon_ring_init()
414 		ring->next_rptr_gpu_addr = rdev->wb.gpu_addr + index;  in radeon_ring_init()
415 		ring->next_rptr_cpu_addr = &rdev->wb.wb[index/4];  in radeon_ring_init()
417 	if (radeon_debugfs_ring_init(rdev, ring)) {  in radeon_ring_init()
420 	radeon_ring_lockup_update(rdev, ring);  in radeon_ring_init()
432 void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_fini()  argument
438 	ring_obj = ring->ring_obj;  in radeon_ring_fini()
439 	ring->ready = false;  in radeon_ring_fini()
440 	ring->ring = NULL;  in radeon_ring_fini()
441 	ring->ring_obj = NULL;  in radeon_ring_fini()
466 	struct radeon_ring *ring = &rdev->ring[ridx];  in radeon_debugfs_ring_info()  local
471 	radeon_ring_free_size(rdev, ring);  in radeon_debugfs_ring_info()
472 	count = (ring->ring_size / 4) - ring->ring_free_dw;  in radeon_debugfs_ring_info()
474 	wptr = radeon_ring_get_wptr(rdev, ring);  in radeon_debugfs_ring_info()
478 	rptr = radeon_ring_get_rptr(rdev, ring);  in radeon_debugfs_ring_info()
482 	if (ring->rptr_save_reg) {  in radeon_debugfs_ring_info()
483 		rptr_next = RREG32(ring->rptr_save_reg);  in radeon_debugfs_ring_info()
485 			   ring->rptr_save_reg, rptr_next, rptr_next);  in radeon_debugfs_ring_info()
490 		   ring->wptr, ring->wptr);  in radeon_debugfs_ring_info()
492 		   ring->last_semaphore_signal_addr);  in radeon_debugfs_ring_info()
494 		   ring->last_semaphore_wait_addr);  in radeon_debugfs_ring_info()
495 	seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);  in radeon_debugfs_ring_info()
498 	if (!ring->ring)  in radeon_debugfs_ring_info()
504 	i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;  in radeon_debugfs_ring_info()
506 		seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);  in radeon_debugfs_ring_info()
512 		i = (i + 1) & ring->ptr_mask;  in radeon_debugfs_ring_info()
539 static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_debugfs_ring_init()  argument
548 		if (&rdev->ring[ridx] != ring)  in radeon_debugfs_ring_init()