129 					  struct radeon_vm *vm,  in radeon_vm_get_bos()  argument
135 	list = drm_malloc_ab(vm->max_pde_used + 2,  in radeon_vm_get_bos()
141 	list[0].robj = vm->page_directory;  in radeon_vm_get_bos()
144 	list[0].tv.bo = &vm->page_directory->tbo;  in radeon_vm_get_bos()
149 	for (i = 0, idx = 1; i <= vm->max_pde_used; i++) {  in radeon_vm_get_bos()
150 		if (!vm->page_tables[i].bo)  in radeon_vm_get_bos()
153 		list[idx].robj = vm->page_tables[i].bo;  in radeon_vm_get_bos()
178 				       struct radeon_vm *vm, int ring)  in radeon_vm_grab_id()  argument
181 	struct radeon_vm_id *vm_id = &vm->ids[ring];  in radeon_vm_grab_id()
237 		     struct radeon_vm *vm,  in radeon_vm_flush()  argument
240 	uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory);  in radeon_vm_flush()
241 	struct radeon_vm_id *vm_id = &vm->ids[ring];  in radeon_vm_flush()
246 		trace_radeon_vm_flush(pd_addr, ring, vm->ids[ring].id);  in radeon_vm_flush()
269 		     struct radeon_vm *vm,  in radeon_vm_fence()  argument
272 	unsigned vm_id = vm->ids[fence->ring].id;  in radeon_vm_fence()
277 	radeon_fence_unref(&vm->ids[fence->ring].last_id_use);  in radeon_vm_fence()
278 	vm->ids[fence->ring].last_id_use = radeon_fence_ref(fence);  in radeon_vm_fence()
293 struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,  in radeon_vm_bo_find()  argument
299 		if (bo_va->vm == vm) {  in radeon_vm_bo_find()
320 				      struct radeon_vm *vm,  in radeon_vm_bo_add()  argument
329 	bo_va->vm = vm;  in radeon_vm_bo_add()
338 	mutex_lock(&vm->mutex);  in radeon_vm_bo_add()
340 	mutex_unlock(&vm->mutex);  in radeon_vm_bo_add()
451 	struct radeon_vm *vm = bo_va->vm;  in radeon_vm_bo_set_addr()  local
476 	mutex_lock(&vm->mutex);  in radeon_vm_bo_set_addr()
481 		it = interval_tree_iter_first(&vm->va, soffset, eoffset);  in radeon_vm_bo_set_addr()
489 			mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
500 			mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
506 		tmp->vm = vm;  in radeon_vm_bo_set_addr()
509 		interval_tree_remove(&bo_va->it, &vm->va);  in radeon_vm_bo_set_addr()
510 		spin_lock(&vm->status_lock);  in radeon_vm_bo_set_addr()
514 		list_add(&tmp->vm_status, &vm->freed);  in radeon_vm_bo_set_addr()
515 		spin_unlock(&vm->status_lock);  in radeon_vm_bo_set_addr()
519 		spin_lock(&vm->status_lock);  in radeon_vm_bo_set_addr()
522 		list_add(&bo_va->vm_status, &vm->cleared);  in radeon_vm_bo_set_addr()
523 		spin_unlock(&vm->status_lock);  in radeon_vm_bo_set_addr()
524 		interval_tree_insert(&bo_va->it, &vm->va);  in radeon_vm_bo_set_addr()
534 	if (eoffset > vm->max_pde_used)  in radeon_vm_bo_set_addr()
535 		vm->max_pde_used = eoffset;  in radeon_vm_bo_set_addr()
543 		if (vm->page_tables[pt_idx].bo)  in radeon_vm_bo_set_addr()
547 		mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
563 		mutex_lock(&vm->mutex);  in radeon_vm_bo_set_addr()
564 		if (vm->page_tables[pt_idx].bo) {  in radeon_vm_bo_set_addr()
566 			mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
568 			mutex_lock(&vm->mutex);  in radeon_vm_bo_set_addr()
572 		vm->page_tables[pt_idx].addr = 0;  in radeon_vm_bo_set_addr()
573 		vm->page_tables[pt_idx].bo = pt;  in radeon_vm_bo_set_addr()
576 	mutex_unlock(&vm->mutex);  in radeon_vm_bo_set_addr()
640 				    struct radeon_vm *vm)  in radeon_vm_update_page_directory()  argument
642 	struct radeon_bo *pd = vm->page_directory;  in radeon_vm_update_page_directory()
654 	ndw += vm->max_pde_used * 6;  in radeon_vm_update_page_directory()
666 	for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {  in radeon_vm_update_page_directory()
667 		struct radeon_bo *bo = vm->page_tables[pt_idx].bo;  in radeon_vm_update_page_directory()
674 		if (vm->page_tables[pt_idx].addr == pt)  in radeon_vm_update_page_directory()
676 		vm->page_tables[pt_idx].addr = pt;  in radeon_vm_update_page_directory()
813 				 struct radeon_vm *vm,  in radeon_vm_update_ptes()  argument
826 		struct radeon_bo *pt = vm->page_tables[pt_idx].bo;  in radeon_vm_update_ptes()
884 static void radeon_vm_fence_pts(struct radeon_vm *vm,  in radeon_vm_fence_pts()  argument
894 		radeon_bo_fence(vm->page_tables[i].bo, fence, true);  in radeon_vm_fence_pts()
914 	struct radeon_vm *vm = bo_va->vm;  in radeon_vm_bo_update()  local
923 			bo_va->bo, vm);  in radeon_vm_bo_update()
927 	spin_lock(&vm->status_lock);  in radeon_vm_bo_update()
930 			spin_unlock(&vm->status_lock);  in radeon_vm_bo_update()
936 		list_add(&bo_va->vm_status, &vm->cleared);  in radeon_vm_bo_update()
938 	spin_unlock(&vm->status_lock);  in radeon_vm_bo_update()
1007 			radeon_sync_fence(&ib.sync, vm->ids[i].last_id_use);  in radeon_vm_bo_update()
1010 	r = radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start,  in radeon_vm_bo_update()
1027 	radeon_vm_fence_pts(vm, bo_va->it.start, bo_va->it.last + 1, ib.fence);  in radeon_vm_bo_update()
1047 			  struct radeon_vm *vm)  in radeon_vm_clear_freed()  argument
1052 	spin_lock(&vm->status_lock);  in radeon_vm_clear_freed()
1053 	while (!list_empty(&vm->freed)) {  in radeon_vm_clear_freed()
1054 		bo_va = list_first_entry(&vm->freed,  in radeon_vm_clear_freed()
1056 		spin_unlock(&vm->status_lock);  in radeon_vm_clear_freed()
1061 		spin_lock(&vm->status_lock);  in radeon_vm_clear_freed()
1068 	spin_unlock(&vm->status_lock);  in radeon_vm_clear_freed()
1085 			     struct radeon_vm *vm)  in radeon_vm_clear_invalids()  argument
1090 	spin_lock(&vm->status_lock);  in radeon_vm_clear_invalids()
1091 	while (!list_empty(&vm->invalidated)) {  in radeon_vm_clear_invalids()
1092 		bo_va = list_first_entry(&vm->invalidated,  in radeon_vm_clear_invalids()
1094 		spin_unlock(&vm->status_lock);  in radeon_vm_clear_invalids()
1100 		spin_lock(&vm->status_lock);  in radeon_vm_clear_invalids()
1102 	spin_unlock(&vm->status_lock);  in radeon_vm_clear_invalids()
1120 	struct radeon_vm *vm = bo_va->vm;  in radeon_vm_bo_rmv()  local
1124 	mutex_lock(&vm->mutex);  in radeon_vm_bo_rmv()
1126 		interval_tree_remove(&bo_va->it, &vm->va);  in radeon_vm_bo_rmv()
1128 	spin_lock(&vm->status_lock);  in radeon_vm_bo_rmv()
1132 		list_add(&bo_va->vm_status, &vm->freed);  in radeon_vm_bo_rmv()
1137 	spin_unlock(&vm->status_lock);  in radeon_vm_bo_rmv()
1139 	mutex_unlock(&vm->mutex);  in radeon_vm_bo_rmv()
1157 		spin_lock(&bo_va->vm->status_lock);  in radeon_vm_bo_invalidate()
1160 			list_add(&bo_va->vm_status, &bo_va->vm->invalidated);  in radeon_vm_bo_invalidate()
1161 		spin_unlock(&bo_va->vm->status_lock);  in radeon_vm_bo_invalidate()
1173 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_init()  argument
1180 	vm->ib_bo_va = NULL;  in radeon_vm_init()
1182 		vm->ids[i].id = 0;  in radeon_vm_init()
1183 		vm->ids[i].flushed_updates = NULL;  in radeon_vm_init()
1184 		vm->ids[i].last_id_use = NULL;  in radeon_vm_init()
1186 	mutex_init(&vm->mutex);  in radeon_vm_init()
1187 	vm->va = RB_ROOT;  in radeon_vm_init()
1188 	spin_lock_init(&vm->status_lock);  in radeon_vm_init()
1189 	INIT_LIST_HEAD(&vm->invalidated);  in radeon_vm_init()
1190 	INIT_LIST_HEAD(&vm->freed);  in radeon_vm_init()
1191 	INIT_LIST_HEAD(&vm->cleared);  in radeon_vm_init()
1198 	vm->page_tables = kzalloc(pts_size, GFP_KERNEL);  in radeon_vm_init()
1199 	if (vm->page_tables == NULL) {  in radeon_vm_init()
1206 			     NULL, &vm->page_directory);  in radeon_vm_init()
1210 	r = radeon_vm_clear_bo(rdev, vm->page_directory);  in radeon_vm_init()
1212 		radeon_bo_unref(&vm->page_directory);  in radeon_vm_init()
1213 		vm->page_directory = NULL;  in radeon_vm_init()
1229 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_fini()  argument
1234 	if (!RB_EMPTY_ROOT(&vm->va)) {  in radeon_vm_fini()
1237 	rbtree_postorder_for_each_entry_safe(bo_va, tmp, &vm->va, it.rb) {  in radeon_vm_fini()
1238 		interval_tree_remove(&bo_va->it, &vm->va);  in radeon_vm_fini()
1247 	list_for_each_entry_safe(bo_va, tmp, &vm->freed, vm_status) {  in radeon_vm_fini()
1254 		radeon_bo_unref(&vm->page_tables[i].bo);  in radeon_vm_fini()
1255 	kfree(vm->page_tables);  in radeon_vm_fini()
1257 	radeon_bo_unref(&vm->page_directory);  in radeon_vm_fini()
1260 		radeon_fence_unref(&vm->ids[i].flushed_updates);  in radeon_vm_fini()
1261 		radeon_fence_unref(&vm->ids[i].last_id_use);  in radeon_vm_fini()
1264 	mutex_destroy(&vm->mutex);  in radeon_vm_fini()