63 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_write()
79 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_read()
110 	(*fence)->seq = ++ring->fence_drv.sync_seq[ring->idx];  in amdgpu_fence_emit()
114 		&ring->fence_drv.fence_queue.lock,  in amdgpu_fence_emit()
117 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit()
132 	mod_timer(&ring->fence_drv.fallback_timer,  in amdgpu_fence_schedule_fallback()
172 	last_seq = atomic64_read(&ring->fence_drv.last_seq);  in amdgpu_fence_activity()
174 		last_emitted = ring->fence_drv.sync_seq[ring->idx];  in amdgpu_fence_activity()
199 	} while (atomic64_xchg(&ring->fence_drv.last_seq, seq) > seq);  in amdgpu_fence_activity()
219 		wake_up_all(&ring->fence_drv.fence_queue);  in amdgpu_fence_process()
251 	if (atomic64_read(&ring->fence_drv.last_seq) >= seq)  in amdgpu_fence_seq_signaled()
256 	if (atomic64_read(&ring->fence_drv.last_seq) >= seq)  in amdgpu_fence_seq_signaled()
277 	if (seq > ring->fence_drv.sync_seq[ring->idx])  in amdgpu_fence_ring_wait_seq()
280 	if (atomic64_read(&ring->fence_drv.last_seq) >= seq)  in amdgpu_fence_ring_wait_seq()
284 	wait_event(ring->fence_drv.fence_queue, (  in amdgpu_fence_ring_wait_seq()
305 	uint64_t seq = atomic64_read(&ring->fence_drv.last_seq) + 1ULL;  in amdgpu_fence_wait_next()
307 	if (seq >= ring->fence_drv.sync_seq[ring->idx])  in amdgpu_fence_wait_next()
325 	uint64_t seq = ring->fence_drv.sync_seq[ring->idx];  in amdgpu_fence_wait_empty()
350 	emitted = ring->fence_drv.sync_seq[ring->idx]  in amdgpu_fence_count_emitted()
351 		- atomic64_read(&ring->fence_drv.last_seq);  in amdgpu_fence_count_emitted()
382 	fdrv = &dst_ring->fence_drv;  in amdgpu_fence_need_sync()
411 	src = &fence->ring->fence_drv;  in amdgpu_fence_note_sync()
412 	dst = &dst_ring->fence_drv;  in amdgpu_fence_note_sync()
442 		ring->fence_drv.cpu_addr = &adev->wb.wb[ring->fence_offs];  in amdgpu_fence_driver_start_ring()
443 		ring->fence_drv.gpu_addr = adev->wb.gpu_addr + (ring->fence_offs * 4);  in amdgpu_fence_driver_start_ring()
447 		ring->fence_drv.cpu_addr = adev->uvd.cpu_addr + index;  in amdgpu_fence_driver_start_ring()
448 		ring->fence_drv.gpu_addr = adev->uvd.gpu_addr + index;  in amdgpu_fence_driver_start_ring()
450 	amdgpu_fence_write(ring, atomic64_read(&ring->fence_drv.last_seq));  in amdgpu_fence_driver_start_ring()
453 	ring->fence_drv.irq_src = irq_src;  in amdgpu_fence_driver_start_ring()
454 	ring->fence_drv.irq_type = irq_type;  in amdgpu_fence_driver_start_ring()
455 	ring->fence_drv.initialized = true;  in amdgpu_fence_driver_start_ring()
459 		 ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);  in amdgpu_fence_driver_start_ring()
476 	ring->fence_drv.cpu_addr = NULL;  in amdgpu_fence_driver_init_ring()
477 	ring->fence_drv.gpu_addr = 0;  in amdgpu_fence_driver_init_ring()
479 		ring->fence_drv.sync_seq[i] = 0;  in amdgpu_fence_driver_init_ring()
481 	atomic64_set(&ring->fence_drv.last_seq, 0);  in amdgpu_fence_driver_init_ring()
482 	ring->fence_drv.initialized = false;  in amdgpu_fence_driver_init_ring()
484 	setup_timer(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback,  in amdgpu_fence_driver_init_ring()
487 	init_waitqueue_head(&ring->fence_drv.fence_queue);  in amdgpu_fence_driver_init_ring()
559 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_fini()
566 		wake_up_all(&ring->fence_drv.fence_queue);  in amdgpu_fence_driver_fini()
567 		amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_fini()
568 			       ring->fence_drv.irq_type);  in amdgpu_fence_driver_fini()
570 		del_timer_sync(&ring->fence_drv.fallback_timer);  in amdgpu_fence_driver_fini()
571 		ring->fence_drv.initialized = false;  in amdgpu_fence_driver_fini()
591 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_suspend()
602 		amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_suspend()
603 			       ring->fence_drv.irq_type);  in amdgpu_fence_driver_suspend()
627 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_resume()
631 		amdgpu_irq_get(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_resume()
632 			       ring->fence_drv.irq_type);  in amdgpu_fence_driver_resume()
651 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_force_completion()
654 		amdgpu_fence_write(ring, ring->fence_drv.sync_seq[i]);  in amdgpu_fence_driver_force_completion()
686 	if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)  in amdgpu_fence_is_signaled()
691 	if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)  in amdgpu_fence_is_signaled()
718 	seq = atomic64_read(&fence->ring->fence_drv.last_seq);  in amdgpu_fence_check_signaled()
726 		__remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake);  in amdgpu_fence_check_signaled()
746 	if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq)  in amdgpu_fence_enable_signaling()
752 	__add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake);  in amdgpu_fence_enable_signaling()
754 	if (!timer_pending(&ring->fence_drv.fallback_timer))  in amdgpu_fence_enable_signaling()
788 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_debugfs_fence_info()
795 			   (unsigned long long)atomic64_read(&ring->fence_drv.last_seq));  in amdgpu_debugfs_fence_info()
797 			   ring->fence_drv.sync_seq[i]);  in amdgpu_debugfs_fence_info()
801 			if (i != j && other && other->fence_drv.initialized &&  in amdgpu_debugfs_fence_info()
802 			    ring->fence_drv.sync_seq[j])  in amdgpu_debugfs_fence_info()
804 					   j, ring->fence_drv.sync_seq[j]);  in amdgpu_debugfs_fence_info()