60 static unsigned radeon_vm_num_pdes(struct radeon_device *rdev)  in radeon_vm_num_pdes()  argument
62 	return rdev->vm_manager.max_pfn >> radeon_vm_block_size;  in radeon_vm_num_pdes()
72 static unsigned radeon_vm_directory_size(struct radeon_device *rdev)  in radeon_vm_directory_size()  argument
74 	return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8);  in radeon_vm_directory_size()
85 int radeon_vm_manager_init(struct radeon_device *rdev)  in radeon_vm_manager_init()  argument
89 	if (!rdev->vm_manager.enabled) {  in radeon_vm_manager_init()
90 		r = radeon_asic_vm_init(rdev);  in radeon_vm_manager_init()
94 		rdev->vm_manager.enabled = true;  in radeon_vm_manager_init()
106 void radeon_vm_manager_fini(struct radeon_device *rdev)  in radeon_vm_manager_fini()  argument
110 	if (!rdev->vm_manager.enabled)  in radeon_vm_manager_fini()
114 		radeon_fence_unref(&rdev->vm_manager.active[i]);  in radeon_vm_manager_fini()
115 	radeon_asic_vm_fini(rdev);  in radeon_vm_manager_fini()
116 	rdev->vm_manager.enabled = false;  in radeon_vm_manager_fini()
128 struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev,  in radeon_vm_get_bos()  argument
177 struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,  in radeon_vm_grab_id()  argument
188 	    vm_id->last_id_use == rdev->vm_manager.active[vm_id->id])  in radeon_vm_grab_id()
195 	for (i = 1; i < rdev->vm_manager.nvm; ++i) {  in radeon_vm_grab_id()
196 		struct radeon_fence *fence = rdev->vm_manager.active[i];  in radeon_vm_grab_id()
215 			return rdev->vm_manager.active[choices[i]];  in radeon_vm_grab_id()
236 void radeon_vm_flush(struct radeon_device *rdev,  in radeon_vm_flush()  argument
250 		radeon_ring_vm_flush(rdev, &rdev->ring[ring],  in radeon_vm_flush()
268 void radeon_vm_fence(struct radeon_device *rdev,  in radeon_vm_fence()  argument
274 	radeon_fence_unref(&rdev->vm_manager.active[vm_id]);  in radeon_vm_fence()
275 	rdev->vm_manager.active[vm_id] = radeon_fence_ref(fence);  in radeon_vm_fence()
319 struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,  in radeon_vm_bo_add()  argument
360 static void radeon_vm_set_pages(struct radeon_device *rdev,  in radeon_vm_set_pages()  argument
369 		uint64_t src = rdev->gart.table_addr + (addr >> 12) * 8;  in radeon_vm_set_pages()
370 		radeon_asic_vm_copy_pages(rdev, ib, pe, src, count);  in radeon_vm_set_pages()
373 		radeon_asic_vm_write_pages(rdev, ib, pe, addr,  in radeon_vm_set_pages()
377 		radeon_asic_vm_set_pages(rdev, ib, pe, addr,  in radeon_vm_set_pages()
388 static int radeon_vm_clear_bo(struct radeon_device *rdev,  in radeon_vm_clear_bo()  argument
407 	r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, 256);  in radeon_vm_clear_bo()
413 	radeon_vm_set_pages(rdev, &ib, addr, 0, entries, 0, 0);  in radeon_vm_clear_bo()
414 	radeon_asic_vm_pad_ib(rdev, &ib);  in radeon_vm_clear_bo()
417 	r = radeon_ib_schedule(rdev, &ib, NULL, false);  in radeon_vm_clear_bo()
425 	radeon_ib_free(rdev, &ib);  in radeon_vm_clear_bo()
446 int radeon_vm_bo_set_addr(struct radeon_device *rdev,  in radeon_vm_bo_set_addr()  argument
466 		if (last_pfn >= rdev->vm_manager.max_pfn) {  in radeon_vm_bo_set_addr()
467 			dev_err(rdev->dev, "va above limit (0x%08X >= 0x%08X)\n",  in radeon_vm_bo_set_addr()
468 				last_pfn, rdev->vm_manager.max_pfn);  in radeon_vm_bo_set_addr()
487 			dev_err(rdev->dev, "bo %p va 0x%010Lx conflict with "  in radeon_vm_bo_set_addr()
535 	BUG_ON(eoffset >= radeon_vm_num_pdes(rdev));  in radeon_vm_bo_set_addr()
552 		r = radeon_bo_create(rdev, RADEON_VM_PTE_COUNT * 8,  in radeon_vm_bo_set_addr()
559 		r = radeon_vm_clear_bo(rdev, pt);  in radeon_vm_bo_set_addr()
597 uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr)  in radeon_vm_map_gart()  argument
602 	result = rdev->gart.pages_entry[addr >> RADEON_GPU_PAGE_SHIFT];  in radeon_vm_map_gart()
642 int radeon_vm_update_page_directory(struct radeon_device *rdev,  in radeon_vm_update_page_directory()  argument
663 	r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);  in radeon_vm_update_page_directory()
686 				radeon_vm_set_pages(rdev, &ib, last_pde,  in radeon_vm_update_page_directory()
700 		radeon_vm_set_pages(rdev, &ib, last_pde, last_pt, count,  in radeon_vm_update_page_directory()
704 		radeon_asic_vm_pad_ib(rdev, &ib);  in radeon_vm_update_page_directory()
706 		radeon_sync_resv(rdev, &ib.sync, pd->tbo.resv, true);  in radeon_vm_update_page_directory()
708 		r = radeon_ib_schedule(rdev, &ib, NULL, false);  in radeon_vm_update_page_directory()
710 			radeon_ib_free(rdev, &ib);  in radeon_vm_update_page_directory()
716 	radeon_ib_free(rdev, &ib);  in radeon_vm_update_page_directory()
733 static void radeon_vm_frag_ptes(struct radeon_device *rdev,  in radeon_vm_frag_ptes()  argument
758 	uint64_t frag_flags = ((rdev->family == CHIP_CAYMAN) ||  in radeon_vm_frag_ptes()
759 			       (rdev->family == CHIP_ARUBA)) ?  in radeon_vm_frag_ptes()
761 	uint64_t frag_align = ((rdev->family == CHIP_CAYMAN) ||  in radeon_vm_frag_ptes()
762 			       (rdev->family == CHIP_ARUBA)) ? 0x200 : 0x80;  in radeon_vm_frag_ptes()
774 		radeon_vm_set_pages(rdev, ib, pe_start, addr, count,  in radeon_vm_frag_ptes()
782 		radeon_vm_set_pages(rdev, ib, pe_start, addr, count,  in radeon_vm_frag_ptes()
789 	radeon_vm_set_pages(rdev, ib, frag_start, addr, count,  in radeon_vm_frag_ptes()
796 		radeon_vm_set_pages(rdev, ib, frag_end, addr, count,  in radeon_vm_frag_ptes()
815 static int radeon_vm_update_ptes(struct radeon_device *rdev,  in radeon_vm_update_ptes()  argument
834 		radeon_sync_resv(rdev, &ib->sync, pt->tbo.resv, true);  in radeon_vm_update_ptes()
850 				radeon_vm_frag_ptes(rdev, ib, last_pte,  in radeon_vm_update_ptes()
867 		radeon_vm_frag_ptes(rdev, ib, last_pte,  in radeon_vm_update_ptes()
913 int radeon_vm_bo_update(struct radeon_device *rdev,  in radeon_vm_bo_update()  argument
925 		dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",  in radeon_vm_bo_update()
951 			addr += rdev->vm_manager.vram_base_offset;  in radeon_vm_bo_update()
996 	r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);  in radeon_vm_bo_update()
1008 	r = radeon_vm_update_ptes(rdev, vm, &ib, bo_va->it.start,  in radeon_vm_bo_update()
1012 		radeon_ib_free(rdev, &ib);  in radeon_vm_bo_update()
1016 	radeon_asic_vm_pad_ib(rdev, &ib);  in radeon_vm_bo_update()
1019 	r = radeon_ib_schedule(rdev, &ib, NULL, false);  in radeon_vm_bo_update()
1021 		radeon_ib_free(rdev, &ib);  in radeon_vm_bo_update()
1028 	radeon_ib_free(rdev, &ib);  in radeon_vm_bo_update()
1044 int radeon_vm_clear_freed(struct radeon_device *rdev,  in radeon_vm_clear_freed()  argument
1056 		r = radeon_vm_bo_update(rdev, bo_va, NULL);  in radeon_vm_clear_freed()
1081 int radeon_vm_clear_invalids(struct radeon_device *rdev,  in radeon_vm_clear_invalids()  argument
1093 		r = radeon_vm_bo_update(rdev, bo_va, NULL);  in radeon_vm_clear_invalids()
1114 void radeon_vm_bo_rmv(struct radeon_device *rdev,  in radeon_vm_bo_rmv()  argument
1148 void radeon_vm_bo_invalidate(struct radeon_device *rdev,  in radeon_vm_bo_invalidate()  argument
1171 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_init()  argument
1190 	pd_size = radeon_vm_directory_size(rdev);  in radeon_vm_init()
1191 	pd_entries = radeon_vm_num_pdes(rdev);  in radeon_vm_init()
1201 	r = radeon_bo_create(rdev, pd_size, align, true,  in radeon_vm_init()
1207 	r = radeon_vm_clear_bo(rdev, vm->page_directory);  in radeon_vm_init()
1226 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)  in radeon_vm_fini()  argument
1232 		dev_err(rdev->dev, "still active bo inside vm\n");  in radeon_vm_fini()
1250 	for (i = 0; i < radeon_vm_num_pdes(rdev); i++)  in radeon_vm_fini()