50 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
51 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);
56 	struct amdgpu_device *adev;  in amdgpu_get_adev()  local
59 	adev = container_of(mman, struct amdgpu_device, mman);  in amdgpu_get_adev()
60 	return adev;  in amdgpu_get_adev()
77 static int amdgpu_ttm_global_init(struct amdgpu_device *adev)  in amdgpu_ttm_global_init()  argument
82 	adev->mman.mem_global_referenced = false;  in amdgpu_ttm_global_init()
83 	global_ref = &adev->mman.mem_global_ref;  in amdgpu_ttm_global_init()
95 	adev->mman.bo_global_ref.mem_glob =  in amdgpu_ttm_global_init()
96 		adev->mman.mem_global_ref.object;  in amdgpu_ttm_global_init()
97 	global_ref = &adev->mman.bo_global_ref.ref;  in amdgpu_ttm_global_init()
105 		drm_global_item_unref(&adev->mman.mem_global_ref);  in amdgpu_ttm_global_init()
109 	adev->mman.mem_global_referenced = true;  in amdgpu_ttm_global_init()
113 static void amdgpu_ttm_global_fini(struct amdgpu_device *adev)  in amdgpu_ttm_global_fini()  argument
115 	if (adev->mman.mem_global_referenced) {  in amdgpu_ttm_global_fini()
116 		drm_global_item_unref(&adev->mman.bo_global_ref.ref);  in amdgpu_ttm_global_fini()
117 		drm_global_item_unref(&adev->mman.mem_global_ref);  in amdgpu_ttm_global_fini()
118 		adev->mman.mem_global_referenced = false;  in amdgpu_ttm_global_fini()
130 	struct amdgpu_device *adev;  in amdgpu_init_mem_type()  local
132 	adev = amdgpu_get_adev(bdev);  in amdgpu_init_mem_type()
143 		man->gpu_offset = adev->mc.gtt_start;  in amdgpu_init_mem_type()
151 		man->gpu_offset = adev->mc.vram_start;  in amdgpu_init_mem_type()
194 		if (rbo->adev->mman.buffer_funcs_ring->ready == false)  in amdgpu_evict_flags()
228 	struct amdgpu_device *adev;  in amdgpu_move_blit()  local
234 	adev = amdgpu_get_adev(bo->bdev);  in amdgpu_move_blit()
235 	ring = adev->mman.buffer_funcs_ring;  in amdgpu_move_blit()
241 		old_start += adev->mc.vram_start;  in amdgpu_move_blit()
244 		old_start += adev->mc.gtt_start;  in amdgpu_move_blit()
252 		new_start += adev->mc.vram_start;  in amdgpu_move_blit()
255 		new_start += adev->mc.gtt_start;  in amdgpu_move_blit()
283 	struct amdgpu_device *adev;  in amdgpu_move_vram_ram()  local
290 	adev = amdgpu_get_adev(bo->bdev);  in amdgpu_move_vram_ram()
330 	struct amdgpu_device *adev;  in amdgpu_move_ram_vram()  local
337 	adev = amdgpu_get_adev(bo->bdev);  in amdgpu_move_ram_vram()
370 	struct amdgpu_device *adev;  in amdgpu_bo_move()  local
374 	adev = amdgpu_get_adev(bo->bdev);  in amdgpu_bo_move()
387 	if (adev->mman.buffer_funcs == NULL ||  in amdgpu_bo_move()
388 	    adev->mman.buffer_funcs_ring == NULL ||  in amdgpu_bo_move()
389 	    !adev->mman.buffer_funcs_ring->ready) {  in amdgpu_bo_move()
415 	atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);  in amdgpu_bo_move()
422 	struct amdgpu_device *adev = amdgpu_get_adev(bdev);  in amdgpu_ttm_io_mem_reserve()  local
440 		if ((mem->bus.offset + mem->bus.size) > adev->mc.visible_vram_size)  in amdgpu_ttm_io_mem_reserve()
442 		mem->bus.base = adev->mc.aper_base;  in amdgpu_ttm_io_mem_reserve()
465 			adev->ddev->hose->dense_mem_base;  in amdgpu_ttm_io_mem_reserve()
483 	struct amdgpu_device		*adev;  member
493 	struct amdgpu_device *adev = amdgpu_get_adev(ttm->bdev);  in amdgpu_ttm_tt_pin_userptr()  local
537 	nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);  in amdgpu_ttm_tt_pin_userptr()
556 	struct amdgpu_device *adev = amdgpu_get_adev(ttm->bdev);  in amdgpu_ttm_tt_unpin_userptr()  local
569 	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);  in amdgpu_ttm_tt_unpin_userptr()
587 	uint32_t flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);  in amdgpu_ttm_backend_bind()
608 	r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,  in amdgpu_ttm_backend_bind()
624 	if (gtt->adev->gart.ready)  in amdgpu_ttm_backend_unbind()
625 		amdgpu_gart_unbind(gtt->adev, gtt->offset, ttm->num_pages);  in amdgpu_ttm_backend_unbind()
651 	struct amdgpu_device *adev;  in amdgpu_ttm_tt_create()  local
654 	adev = amdgpu_get_adev(bdev);  in amdgpu_ttm_tt_create()
661 	gtt->adev = adev;  in amdgpu_ttm_tt_create()
671 	struct amdgpu_device *adev;  in amdgpu_ttm_tt_populate()  local
697 	adev = amdgpu_get_adev(ttm->bdev);  in amdgpu_ttm_tt_populate()
701 		return ttm_dma_populate(&gtt->ttm, adev->dev);  in amdgpu_ttm_tt_populate()
711 		gtt->ttm.dma_address[i] = pci_map_page(adev->pdev, ttm->pages[i],  in amdgpu_ttm_tt_populate()
714 		if (pci_dma_mapping_error(adev->pdev, gtt->ttm.dma_address[i])) {  in amdgpu_ttm_tt_populate()
716 				pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],  in amdgpu_ttm_tt_populate()
729 	struct amdgpu_device *adev;  in amdgpu_ttm_tt_unpopulate()  local
743 	adev = amdgpu_get_adev(ttm->bdev);  in amdgpu_ttm_tt_unpopulate()
747 		ttm_dma_unpopulate(&gtt->ttm, adev->dev);  in amdgpu_ttm_tt_unpopulate()
754 			pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],  in amdgpu_ttm_tt_unpopulate()
815 uint32_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,  in amdgpu_ttm_tt_pte_flags()  argument
830 	if (adev->asic_type >= CHIP_TONGA)  in amdgpu_ttm_tt_pte_flags()
856 int amdgpu_ttm_init(struct amdgpu_device *adev)  in amdgpu_ttm_init()  argument
860 	r = amdgpu_ttm_global_init(adev);  in amdgpu_ttm_init()
865 	r = ttm_bo_device_init(&adev->mman.bdev,  in amdgpu_ttm_init()
866 			       adev->mman.bo_global_ref.ref.object,  in amdgpu_ttm_init()
868 			       adev->ddev->anon_inode->i_mapping,  in amdgpu_ttm_init()
870 			       adev->need_dma32);  in amdgpu_ttm_init()
875 	adev->mman.initialized = true;  in amdgpu_ttm_init()
876 	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,  in amdgpu_ttm_init()
877 				adev->mc.real_vram_size >> PAGE_SHIFT);  in amdgpu_ttm_init()
883 	amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);  in amdgpu_ttm_init()
885 	r = amdgpu_bo_create(adev, 256 * 1024, PAGE_SIZE, true,  in amdgpu_ttm_init()
888 			     NULL, NULL, &adev->stollen_vga_memory);  in amdgpu_ttm_init()
892 	r = amdgpu_bo_reserve(adev->stollen_vga_memory, false);  in amdgpu_ttm_init()
895 	r = amdgpu_bo_pin(adev->stollen_vga_memory, AMDGPU_GEM_DOMAIN_VRAM, NULL);  in amdgpu_ttm_init()
896 	amdgpu_bo_unreserve(adev->stollen_vga_memory);  in amdgpu_ttm_init()
898 		amdgpu_bo_unref(&adev->stollen_vga_memory);  in amdgpu_ttm_init()
902 		 (unsigned) (adev->mc.real_vram_size / (1024 * 1024)));  in amdgpu_ttm_init()
903 	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT,  in amdgpu_ttm_init()
904 				adev->mc.gtt_size >> PAGE_SHIFT);  in amdgpu_ttm_init()
910 		 (unsigned)(adev->mc.gtt_size / (1024 * 1024)));  in amdgpu_ttm_init()
912 	adev->gds.mem.total_size = adev->gds.mem.total_size << AMDGPU_GDS_SHIFT;  in amdgpu_ttm_init()
913 	adev->gds.mem.gfx_partition_size = adev->gds.mem.gfx_partition_size << AMDGPU_GDS_SHIFT;  in amdgpu_ttm_init()
914 	adev->gds.mem.cs_partition_size = adev->gds.mem.cs_partition_size << AMDGPU_GDS_SHIFT;  in amdgpu_ttm_init()
915 	adev->gds.gws.total_size = adev->gds.gws.total_size << AMDGPU_GWS_SHIFT;  in amdgpu_ttm_init()
916 	adev->gds.gws.gfx_partition_size = adev->gds.gws.gfx_partition_size << AMDGPU_GWS_SHIFT;  in amdgpu_ttm_init()
917 	adev->gds.gws.cs_partition_size = adev->gds.gws.cs_partition_size << AMDGPU_GWS_SHIFT;  in amdgpu_ttm_init()
918 	adev->gds.oa.total_size = adev->gds.oa.total_size << AMDGPU_OA_SHIFT;  in amdgpu_ttm_init()
919 	adev->gds.oa.gfx_partition_size = adev->gds.oa.gfx_partition_size << AMDGPU_OA_SHIFT;  in amdgpu_ttm_init()
920 	adev->gds.oa.cs_partition_size = adev->gds.oa.cs_partition_size << AMDGPU_OA_SHIFT;  in amdgpu_ttm_init()
922 	r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,  in amdgpu_ttm_init()
923 				adev->gds.mem.total_size >> PAGE_SHIFT);  in amdgpu_ttm_init()
930 	r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,  in amdgpu_ttm_init()
931 				adev->gds.gws.total_size >> PAGE_SHIFT);  in amdgpu_ttm_init()
938 	r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,  in amdgpu_ttm_init()
939 				adev->gds.oa.total_size >> PAGE_SHIFT);  in amdgpu_ttm_init()
945 	r = amdgpu_ttm_debugfs_init(adev);  in amdgpu_ttm_init()
953 void amdgpu_ttm_fini(struct amdgpu_device *adev)  in amdgpu_ttm_fini()  argument
957 	if (!adev->mman.initialized)  in amdgpu_ttm_fini()
959 	amdgpu_ttm_debugfs_fini(adev);  in amdgpu_ttm_fini()
960 	if (adev->stollen_vga_memory) {  in amdgpu_ttm_fini()
961 		r = amdgpu_bo_reserve(adev->stollen_vga_memory, false);  in amdgpu_ttm_fini()
963 			amdgpu_bo_unpin(adev->stollen_vga_memory);  in amdgpu_ttm_fini()
964 			amdgpu_bo_unreserve(adev->stollen_vga_memory);  in amdgpu_ttm_fini()
966 		amdgpu_bo_unref(&adev->stollen_vga_memory);  in amdgpu_ttm_fini()
968 	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);  in amdgpu_ttm_fini()
969 	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);  in amdgpu_ttm_fini()
970 	ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);  in amdgpu_ttm_fini()
971 	ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);  in amdgpu_ttm_fini()
972 	ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);  in amdgpu_ttm_fini()
973 	ttm_bo_device_release(&adev->mman.bdev);  in amdgpu_ttm_fini()
974 	amdgpu_gart_fini(adev);  in amdgpu_ttm_fini()
975 	amdgpu_ttm_global_fini(adev);  in amdgpu_ttm_fini()
976 	adev->mman.initialized = false;  in amdgpu_ttm_fini()
982 void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size)  in amdgpu_ttm_set_active_vram_size()  argument
986 	if (!adev->mman.initialized)  in amdgpu_ttm_set_active_vram_size()
989 	man = &adev->mman.bdev.man[TTM_PL_VRAM];  in amdgpu_ttm_set_active_vram_size()
997 	struct amdgpu_device *adev;  in amdgpu_mmap()  local
1003 	adev = file_priv->minor->dev->dev_private;  in amdgpu_mmap()
1004 	if (adev == NULL)  in amdgpu_mmap()
1007 	return ttm_bo_mmap(filp, vma, &adev->mman.bdev);  in amdgpu_mmap()
1017 	struct amdgpu_device *adev = ring->adev;  in amdgpu_copy_buffer()  local
1024 	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;  in amdgpu_copy_buffer()
1026 	num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw;  in amdgpu_copy_buffer()
1045 		r = amdgpu_sync_resv(adev, &ib->sync, resv,  in amdgpu_copy_buffer()
1056 		amdgpu_emit_copy_buffer(adev, ib, src_offset, dst_offset,  in amdgpu_copy_buffer()
1064 	amdgpu_vm_pad_ib(adev, ib);  in amdgpu_copy_buffer()
1066 	r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1,  in amdgpu_copy_buffer()
1074 		amdgpu_ib_free(adev, ib);  in amdgpu_copy_buffer()
1079 	amdgpu_ib_free(adev, ib);  in amdgpu_copy_buffer()
1091 	struct amdgpu_device *adev = dev->dev_private;  in amdgpu_mm_dump_table()  local
1092 	struct drm_mm *mm = (struct drm_mm *)adev->mman.bdev.man[ttm_pl].priv;  in amdgpu_mm_dump_table()
1094 	struct ttm_bo_global *glob = adev->mman.bdev.glob;  in amdgpu_mm_dump_table()
1101 			   adev->mman.bdev.man[ttm_pl].size,  in amdgpu_mm_dump_table()
1102 			   (u64)atomic64_read(&adev->vram_usage) >> 20,  in amdgpu_mm_dump_table()
1103 			   (u64)atomic64_read(&adev->vram_vis_usage) >> 20);  in amdgpu_mm_dump_table()
1122 	struct amdgpu_device *adev = f->f_inode->i_private;  in amdgpu_ttm_vram_read()  local
1133 		if (*pos >= adev->mc.mc_vram_size)  in amdgpu_ttm_vram_read()
1136 		spin_lock_irqsave(&adev->mmio_idx_lock, flags);  in amdgpu_ttm_vram_read()
1140 		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);  in amdgpu_ttm_vram_read()
1164 	struct amdgpu_device *adev = f->f_inode->i_private;  in amdgpu_ttm_gtt_read()  local
1175 		if (p >= adev->gart.num_cpu_pages)  in amdgpu_ttm_gtt_read()
1178 		page = adev->gart.pages[p];  in amdgpu_ttm_gtt_read()
1184 			kunmap(adev->gart.pages[p]);  in amdgpu_ttm_gtt_read()
1208 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)  in amdgpu_ttm_debugfs_init()  argument
1213 	struct drm_minor *minor = adev->ddev->primary;  in amdgpu_ttm_debugfs_init()
1217 				  adev, &amdgpu_ttm_vram_fops);  in amdgpu_ttm_debugfs_init()
1220 	i_size_write(ent->d_inode, adev->mc.mc_vram_size);  in amdgpu_ttm_debugfs_init()
1221 	adev->mman.vram = ent;  in amdgpu_ttm_debugfs_init()
1224 				  adev, &amdgpu_ttm_gtt_fops);  in amdgpu_ttm_debugfs_init()
1227 	i_size_write(ent->d_inode, adev->mc.gtt_size);  in amdgpu_ttm_debugfs_init()
1228 	adev->mman.gtt = ent;  in amdgpu_ttm_debugfs_init()
1237 	return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);  in amdgpu_ttm_debugfs_init()
1244 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)  in amdgpu_ttm_debugfs_fini()  argument
1248 	debugfs_remove(adev->mman.vram);  in amdgpu_ttm_debugfs_fini()
1249 	adev->mman.vram = NULL;  in amdgpu_ttm_debugfs_fini()
1251 	debugfs_remove(adev->mman.gtt);  in amdgpu_ttm_debugfs_fini()
1252 	adev->mman.gtt = NULL;  in amdgpu_ttm_debugfs_fini()