60 static unsigned radeon_vm_num_pdes(struct radeon_device *rdev)  in radeon_vm_num_pdes()  argument
62 	return rdev->vm_manager.max_pfn >> radeon_vm_block_size;  in radeon_vm_num_pdes()
72 static unsigned radeon_vm_directory_size(struct radeon_device *rdev)  in radeon_vm_directory_size()  argument
74 	return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8);  in radeon_vm_directory_size()
85 int radeon_vm_manager_init(struct radeon_device *rdev)  in radeon_vm_manager_init()  argument
89 	if (!rdev->vm_manager.enabled) {  in radeon_vm_manager_init()
90 		r = radeon_asic_vm_init(rdev);  in radeon_vm_manager_init()
94 		rdev->vm_manager.enabled = true;  in radeon_vm_manager_init()
106 void radeon_vm_manager_fini(struct radeon_device *rdev)  in radeon_vm_manager_fini()  argument
110 	if (!rdev->vm_manager.enabled)  in radeon_vm_manager_fini()
114 		radeon_fence_unref(&rdev->vm_manager.active[i]);  in radeon_vm_manager_fini()
115 	radeon_asic_vm_fini(rdev);  in radeon_vm_manager_fini()
116 	rdev->vm_manager.enabled = false;  in radeon_vm_manager_fini()
128 struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev,  in radeon_vm_get_bos()  argument
177 struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,  in radeon_vm_grab_id()  argument
188 	    vm_id->last_id_use == rdev->vm_manager.active[vm_id->id])  in radeon_vm_grab_id()
195 	for (i = 1; i < rdev->vm_manager.nvm; ++i) {  in radeon_vm_grab_id()
196 		struct radeon_fence *fence = rdev->vm_manager.active[i];  in radeon_vm_grab_id()
215 			return rdev->vm_manager.active[choices[i]];  in radeon_vm_grab_id()
236 void radeon_vm_flush(struct radeon_device *rdev,  in radeon_vm_flush()  argument
250 		radeon_ring_vm_flush(rdev, &rdev->ring[ring],  in radeon_vm_flush()
268 void radeon_vm_fence(struct radeon_device *rdev,  in radeon_vm_fence()  argument
274 	radeon_fence_unref(&rdev->vm_manager.active[vm_id]);  in radeon_vm_fence()
275 	rdev->vm_manager.active[vm_id] = radeon_fence_ref(fence);  in radeon_vm_fence()
319 struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,  in radeon_vm_bo_add()  argument
359 static void radeon_vm_set_pages(struct radeon_device *rdev,  in radeon_vm_set_pages()  argument
368 		uint64_t src = rdev->gart.table_addr + (addr >> 12) * 8;  in radeon_vm_set_pages()
369 		radeon_asic_vm_copy_pages(rdev, ib, pe, src, count);  in radeon_vm_set_pages()
372 		radeon_asic_vm_write_pages(rdev, ib, pe, addr,  in radeon_vm_set_pages()
376 		radeon_asic_vm_set_pages(rdev, ib, pe, addr,  in radeon_vm_set_pages()
387 static int radeon_vm_clear_bo(struct radeon_device *rdev,  in radeon_vm_clear_bo()  argument
406 	r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, 256);  in radeon_vm_clear_bo()
412 	radeon_vm_set_pages(rdev, &ib, addr, 0, entries, 0, 0);  in radeon_vm_clear_bo()
413 	radeon_asic_vm_pad_ib(rdev, &ib);  in radeon_vm_clear_bo()
416 	r = radeon_ib_schedule(rdev, &ib, NULL, false);  in radeon_vm_clear_bo()
424 	radeon_ib_free(rdev, &ib);  in radeon_vm_clear_bo()
445 int radeon_vm_bo_set_addr(struct radeon_device *rdev,  in radeon_vm_bo_set_addr()  argument
465 		if (last_pfn >= rdev->vm_manager.max_pfn) {  in radeon_vm_bo_set_addr()
466 			dev_err(rdev->dev, "va above limit (0x%08X >= 0x%08X)\n",  in radeon_vm_bo_set_addr()
467 				last_pfn, rdev->vm_manager.max_pfn);  in radeon_vm_bo_set_addr()
486 			dev_err(rdev->dev, "bo %p va 0x%010Lx conflict with "  in radeon_vm_bo_set_addr()
532 	BUG_ON(eoffset >= radeon_vm_num_pdes(rdev));  in radeon_vm_bo_set_addr()
549 		r = radeon_bo_create(rdev, RADEON_VM_PTE_COUNT * 8,  in radeon_vm_bo_set_addr()
556 		r = radeon_vm_clear_bo(rdev, pt);  in radeon_vm_bo_set_addr()
594 uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr)  in radeon_vm_map_gart()  argument
599 	result = rdev->gart.pages_entry[addr >> RADEON_GPU_PAGE_SHIFT];  in radeon_vm_map_gart()
639 int radeon_vm_update_page_directory(struct radeon_device *rdev,  in radeon_vm_update_page_directory()  argument
660 	r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);  in radeon_vm_update_page_directory()
683 				radeon_vm_set_pages(rdev, &ib, last_pde,  in radeon_vm_update_page_directory()
697 		radeon_vm_set_pages(rdev, &ib, last_pde, last_pt, count,  in radeon_vm_update_page_directory()
701 		radeon_asic_vm_pad_ib(rdev, &ib);  in radeon_vm_update_page_directory()
703 		radeon_sync_resv(rdev, &ib.sync, pd->tbo.resv, true);  in radeon_vm_update_page_directory()
705 		r = radeon_ib_schedule(rdev, &ib, NULL, false);  in radeon_vm_update_page_directory()
707 			radeon_ib_free(rdev, &ib);  in radeon_vm_update_page_directory()
713 	radeon_ib_free(rdev, &ib);  in radeon_vm_update_page_directory()
730 static void radeon_vm_frag_ptes(struct radeon_device *rdev,  in radeon_vm_frag_ptes()  argument
755 	uint64_t frag_flags = ((rdev->family == CHIP_CAYMAN) ||  in radeon_vm_frag_ptes()
756 			       (rdev->family == CHIP_ARUBA)) ?  in radeon_vm_frag_ptes()
758 	uint64_t frag_align = ((rdev->family == CHIP_CAYMAN) ||  in radeon_vm_frag_ptes()
759 			       (rdev->family == CHIP_ARUBA)) ? 0x200 : 0x80;  in radeon_vm_frag_ptes()
771 		radeon_vm_set_pages(rdev, ib, pe_start, addr, count,  in radeon_vm_frag_ptes()
779 		radeon_vm_set_pages(rdev, ib, pe_start, addr, count,  in radeon_vm_frag_ptes()
786 	radeon_vm_set_pages(rdev, ib, frag_start, addr, count,  in radeon_vm_frag_ptes()
793 		radeon_vm_set_pages(rdev, ib, frag_end, addr, count,  in radeon_vm_frag_ptes()
812 static int radeon_vm_update_ptes(struct radeon_device *rdev,  in radeon_vm_update_ptes()  argument
831 		radeon_sync_resv(rdev, &ib->sync, pt->tbo.resv, true);  in radeon_vm_update_ptes()
847 				radeon_vm_frag_ptes(rdev, ib, last_pte,  in radeon_vm_update_ptes()
864 		radeon_vm_frag_ptes(rdev, ib, last_pte,  in radeon_vm_update_ptes()
910 int radeon_vm_bo_update(struct radeon_device *rdev,  in radeon_vm_bo_update()  argument
922 		dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",  in radeon_vm_bo_update()
957 			addr += rdev->vm_manager.vram_base_offset;  in radeon_vm_bo_update()
998 	r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);  in radeon_vm_bo_update()
1010 	r = radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start,  in radeon_vm_bo_update()
1014 		radeon_ib_free(rdev, &ib);  in radeon_vm_bo_update()
1018 	radeon_asic_vm_pad_ib(rdev, &ib);  in radeon_vm_bo_update()
1021 	r = radeon_ib_schedule(rdev, &ib, NULL, false);  in radeon_vm_bo_update()
1023 		radeon_ib_free(rdev, &ib);  in radeon_vm_bo_update()
1030 	radeon_ib_free(rdev, &ib);  in radeon_vm_bo_update()
1046 int radeon_vm_clear_freed(struct radeon_device *rdev,  in radeon_vm_clear_freed()  argument
1058 		r = radeon_vm_bo_update(rdev, bo_va, NULL);  in radeon_vm_clear_freed()
1084 int radeon_vm_clear_invalids(struct radeon_device *rdev,  in radeon_vm_clear_invalids()  argument
1096 		r = radeon_vm_bo_update(rdev, bo_va, NULL);  in radeon_vm_clear_invalids()
1117 void radeon_vm_bo_rmv(struct radeon_device *rdev,  in radeon_vm_bo_rmv()  argument
1151 void radeon_vm_bo_invalidate(struct radeon_device *rdev,  in radeon_vm_bo_invalidate()  argument
1173 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_init()  argument
1193 	pd_size = radeon_vm_directory_size(rdev);  in radeon_vm_init()
1194 	pd_entries = radeon_vm_num_pdes(rdev);  in radeon_vm_init()
1204 	r = radeon_bo_create(rdev, pd_size, align, true,  in radeon_vm_init()
1210 	r = radeon_vm_clear_bo(rdev, vm->page_directory);  in radeon_vm_init()
1229 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_fini()  argument
1235 		dev_err(rdev->dev, "still active bo inside vm\n");  in radeon_vm_fini()
1253 	for (i = 0; i < radeon_vm_num_pdes(rdev); i++)  in radeon_vm_fini()