49 static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring);
59 void amdgpu_ring_free_size(struct amdgpu_ring *ring)  in amdgpu_ring_free_size()  argument
61 	uint32_t rptr = amdgpu_ring_get_rptr(ring);  in amdgpu_ring_free_size()
64 	ring->ring_free_dw = rptr + (ring->ring_size / 4);  in amdgpu_ring_free_size()
65 	ring->ring_free_dw -= ring->wptr;  in amdgpu_ring_free_size()
66 	ring->ring_free_dw &= ring->ptr_mask;  in amdgpu_ring_free_size()
67 	if (!ring->ring_free_dw) {  in amdgpu_ring_free_size()
69 		ring->ring_free_dw = ring->ring_size / 4;  in amdgpu_ring_free_size()
83 int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw)  in amdgpu_ring_alloc()  argument
88 	if (ndw > (ring->ring_size / 4))  in amdgpu_ring_alloc()
92 	amdgpu_ring_free_size(ring);  in amdgpu_ring_alloc()
93 	ndw = (ndw + ring->align_mask) & ~ring->align_mask;  in amdgpu_ring_alloc()
94 	while (ndw > (ring->ring_free_dw - 1)) {  in amdgpu_ring_alloc()
95 		amdgpu_ring_free_size(ring);  in amdgpu_ring_alloc()
96 		if (ndw < ring->ring_free_dw) {  in amdgpu_ring_alloc()
99 		r = amdgpu_fence_wait_next(ring);  in amdgpu_ring_alloc()
103 	ring->count_dw = ndw;  in amdgpu_ring_alloc()
104 	ring->wptr_old = ring->wptr;  in amdgpu_ring_alloc()
119 int amdgpu_ring_lock(struct amdgpu_ring *ring, unsigned ndw)  in amdgpu_ring_lock()  argument
123 	mutex_lock(ring->ring_lock);  in amdgpu_ring_lock()
124 	r = amdgpu_ring_alloc(ring, ndw);  in amdgpu_ring_lock()
126 		mutex_unlock(ring->ring_lock);  in amdgpu_ring_lock()
139 void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)  in amdgpu_ring_insert_nop()  argument
144 		amdgpu_ring_write(ring, ring->nop);  in amdgpu_ring_insert_nop()
157 void amdgpu_ring_commit(struct amdgpu_ring *ring)  in amdgpu_ring_commit()  argument
162 	count = ring->align_mask + 1 - (ring->wptr & ring->align_mask);  in amdgpu_ring_commit()
163 	count %= ring->align_mask + 1;  in amdgpu_ring_commit()
164 	ring->funcs->insert_nop(ring, count);  in amdgpu_ring_commit()
167 	amdgpu_ring_set_wptr(ring);  in amdgpu_ring_commit()
178 void amdgpu_ring_unlock_commit(struct amdgpu_ring *ring)  in amdgpu_ring_unlock_commit()  argument
180 	amdgpu_ring_commit(ring);  in amdgpu_ring_unlock_commit()
181 	mutex_unlock(ring->ring_lock);  in amdgpu_ring_unlock_commit()
191 void amdgpu_ring_undo(struct amdgpu_ring *ring)  in amdgpu_ring_undo()  argument
193 	ring->wptr = ring->wptr_old;  in amdgpu_ring_undo()
203 void amdgpu_ring_unlock_undo(struct amdgpu_ring *ring)  in amdgpu_ring_unlock_undo()  argument
205 	amdgpu_ring_undo(ring);  in amdgpu_ring_unlock_undo()
206 	mutex_unlock(ring->ring_lock);  in amdgpu_ring_unlock_undo()
216 unsigned amdgpu_ring_backup(struct amdgpu_ring *ring,  in amdgpu_ring_backup()  argument
222 	mutex_lock(ring->ring_lock);  in amdgpu_ring_backup()
225 	if (ring->ring_obj == NULL) {  in amdgpu_ring_backup()
226 		mutex_unlock(ring->ring_lock);  in amdgpu_ring_backup()
231 	if (!amdgpu_fence_count_emitted(ring)) {  in amdgpu_ring_backup()
232 		mutex_unlock(ring->ring_lock);  in amdgpu_ring_backup()
236 	ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);  in amdgpu_ring_backup()
238 	size = ring->wptr + (ring->ring_size / 4);  in amdgpu_ring_backup()
240 	size &= ring->ptr_mask;  in amdgpu_ring_backup()
242 		mutex_unlock(ring->ring_lock);  in amdgpu_ring_backup()
249 		mutex_unlock(ring->ring_lock);  in amdgpu_ring_backup()
253 		(*data)[i] = ring->ring[ptr++];  in amdgpu_ring_backup()
254 		ptr &= ring->ptr_mask;  in amdgpu_ring_backup()
257 	mutex_unlock(ring->ring_lock);  in amdgpu_ring_backup()
270 int amdgpu_ring_restore(struct amdgpu_ring *ring,  in amdgpu_ring_restore()  argument
279 	r = amdgpu_ring_lock(ring, size);  in amdgpu_ring_restore()
284 		amdgpu_ring_write(ring, data[i]);  in amdgpu_ring_restore()
287 	amdgpu_ring_unlock_commit(ring);  in amdgpu_ring_restore()
303 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,  in amdgpu_ring_init()  argument
311 	if (ring->adev == NULL) {  in amdgpu_ring_init()
315 		ring->adev = adev;  in amdgpu_ring_init()
316 		ring->idx = adev->num_rings++;  in amdgpu_ring_init()
317 		adev->rings[ring->idx] = ring;  in amdgpu_ring_init()
318 		r = amdgpu_fence_driver_init_ring(ring);  in amdgpu_ring_init()
323 	r = amdgpu_wb_get(adev, &ring->rptr_offs);  in amdgpu_ring_init()
329 	r = amdgpu_wb_get(adev, &ring->wptr_offs);  in amdgpu_ring_init()
335 	r = amdgpu_wb_get(adev, &ring->fence_offs);  in amdgpu_ring_init()
341 	r = amdgpu_wb_get(adev, &ring->next_rptr_offs);  in amdgpu_ring_init()
346 	ring->next_rptr_gpu_addr = adev->wb.gpu_addr + (ring->next_rptr_offs * 4);  in amdgpu_ring_init()
347 	ring->next_rptr_cpu_addr = &adev->wb.wb[ring->next_rptr_offs];  in amdgpu_ring_init()
348 	spin_lock_init(&ring->fence_lock);  in amdgpu_ring_init()
349 	r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type);  in amdgpu_ring_init()
355 	ring->ring_lock = &adev->ring_lock;  in amdgpu_ring_init()
359 	ring->ring_size = ring_size;  in amdgpu_ring_init()
360 	ring->align_mask = align_mask;  in amdgpu_ring_init()
361 	ring->nop = nop;  in amdgpu_ring_init()
362 	ring->type = ring_type;  in amdgpu_ring_init()
365 	if (ring->ring_obj == NULL) {  in amdgpu_ring_init()
366 		r = amdgpu_bo_create(adev, ring->ring_size, PAGE_SIZE, true,  in amdgpu_ring_init()
368 				     NULL, NULL, &ring->ring_obj);  in amdgpu_ring_init()
373 		r = amdgpu_bo_reserve(ring->ring_obj, false);  in amdgpu_ring_init()
376 		r = amdgpu_bo_pin(ring->ring_obj, AMDGPU_GEM_DOMAIN_GTT,  in amdgpu_ring_init()
377 					&ring->gpu_addr);  in amdgpu_ring_init()
379 			amdgpu_bo_unreserve(ring->ring_obj);  in amdgpu_ring_init()
383 		r = amdgpu_bo_kmap(ring->ring_obj,  in amdgpu_ring_init()
384 				       (void **)&ring->ring);  in amdgpu_ring_init()
385 		amdgpu_bo_unreserve(ring->ring_obj);  in amdgpu_ring_init()
391 	ring->ptr_mask = (ring->ring_size / 4) - 1;  in amdgpu_ring_init()
392 	ring->ring_free_dw = ring->ring_size / 4;  in amdgpu_ring_init()
394 	if (amdgpu_debugfs_ring_init(adev, ring)) {  in amdgpu_ring_init()
408 void amdgpu_ring_fini(struct amdgpu_ring *ring)  in amdgpu_ring_fini()  argument
413 	if (ring->ring_lock == NULL)  in amdgpu_ring_fini()
416 	mutex_lock(ring->ring_lock);  in amdgpu_ring_fini()
417 	ring_obj = ring->ring_obj;  in amdgpu_ring_fini()
418 	ring->ready = false;  in amdgpu_ring_fini()
419 	ring->ring = NULL;  in amdgpu_ring_fini()
420 	ring->ring_obj = NULL;  in amdgpu_ring_fini()
421 	mutex_unlock(ring->ring_lock);  in amdgpu_ring_fini()
423 	amdgpu_wb_free(ring->adev, ring->fence_offs);  in amdgpu_ring_fini()
424 	amdgpu_wb_free(ring->adev, ring->rptr_offs);  in amdgpu_ring_fini()
425 	amdgpu_wb_free(ring->adev, ring->wptr_offs);  in amdgpu_ring_fini()
426 	amdgpu_wb_free(ring->adev, ring->next_rptr_offs);  in amdgpu_ring_fini()
458 		return a_fence->ring;  in amdgpu_ring_from_fence()
474 	struct amdgpu_ring *ring = (void *)(((uint8_t*)adev) + roffset);  in amdgpu_debugfs_ring_info()  local
479 	amdgpu_ring_free_size(ring);  in amdgpu_debugfs_ring_info()
480 	count = (ring->ring_size / 4) - ring->ring_free_dw;  in amdgpu_debugfs_ring_info()
482 	wptr = amdgpu_ring_get_wptr(ring);  in amdgpu_debugfs_ring_info()
486 	rptr = amdgpu_ring_get_rptr(ring);  in amdgpu_debugfs_ring_info()
493 		   ring->wptr, ring->wptr);  in amdgpu_debugfs_ring_info()
495 		   ring->last_semaphore_signal_addr);  in amdgpu_debugfs_ring_info()
497 		   ring->last_semaphore_wait_addr);  in amdgpu_debugfs_ring_info()
498 	seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);  in amdgpu_debugfs_ring_info()
501 	if (!ring->ready)  in amdgpu_debugfs_ring_info()
507 	i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;  in amdgpu_debugfs_ring_info()
509 		seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);  in amdgpu_debugfs_ring_info()
515 		i = (i + 1) & ring->ptr_mask;  in amdgpu_debugfs_ring_info()
524 static int amdgpu_dma1_index = offsetof(struct amdgpu_device, sdma.instance[0].ring);
525 static int amdgpu_dma2_index = offsetof(struct amdgpu_device, sdma.instance[1].ring);
526 static int r600_uvd_index = offsetof(struct amdgpu_device, uvd.ring);
527 static int si_vce1_index = offsetof(struct amdgpu_device, vce.ring[0]);
528 static int si_vce2_index = offsetof(struct amdgpu_device, vce.ring[1]);
543 static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring)  in amdgpu_debugfs_ring_init()  argument
553 		if (other != ring)  in amdgpu_debugfs_ring_init()