134 nouveau_bo_del_ttm(struct ttm_buffer_object *bo)  in nouveau_bo_del_ttm()  argument
136 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);  in nouveau_bo_del_ttm()
138 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_del_ttm()
141 		DRM_ERROR("bo %p still attached to GEM object\n", bo);  in nouveau_bo_del_ttm()
151 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_fixup_align()
215 	nvbo->bo.bdev = &drm->ttm.bdev;  in nouveau_bo_new()
227 	nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;  in nouveau_bo_new()
233 	ret = ttm_bo_init(&drm->ttm.bdev, &nvbo->bo, size,  in nouveau_bo_new()
262 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in set_placement_range()
268 	    nvbo->bo.mem.num_pages < vram_pages / 4) {  in set_placement_range()
315 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_pin()
316 	struct ttm_buffer_object *bo = &nvbo->bo;  in nouveau_bo_pin()  local
320 	ret = ttm_bo_reserve(bo, false, false, false, NULL);  in nouveau_bo_pin()
327 			if (bo->mem.mem_type == TTM_PL_VRAM) {  in nouveau_bo_pin()
328 				struct nvkm_mem *mem = bo->mem.mm_node;  in nouveau_bo_pin()
338 		if (!(memtype & (1 << bo->mem.mem_type)) || evict) {  in nouveau_bo_pin()
340 				      "0x%08x vs 0x%08x\n", bo,  in nouveau_bo_pin()
341 				 1 << bo->mem.mem_type, memtype);  in nouveau_bo_pin()
368 	switch (bo->mem.mem_type) {  in nouveau_bo_pin()
370 		drm->gem.vram_available -= bo->mem.size;  in nouveau_bo_pin()
373 		drm->gem.gart_available -= bo->mem.size;  in nouveau_bo_pin()
382 	ttm_bo_unreserve(bo);  in nouveau_bo_pin()
389 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_unpin()
390 	struct ttm_buffer_object *bo = &nvbo->bo;  in nouveau_bo_unpin()  local
393 	ret = ttm_bo_reserve(bo, false, false, false, NULL);  in nouveau_bo_unpin()
402 	nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);  in nouveau_bo_unpin()
406 		switch (bo->mem.mem_type) {  in nouveau_bo_unpin()
408 			drm->gem.vram_available += bo->mem.size;  in nouveau_bo_unpin()
411 			drm->gem.gart_available += bo->mem.size;  in nouveau_bo_unpin()
419 	ttm_bo_unreserve(bo);  in nouveau_bo_unpin()
428 	ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL);  in nouveau_bo_map()
437 		ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,  in nouveau_bo_map()
440 	ttm_bo_unreserve(&nvbo->bo);  in nouveau_bo_map()
461 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_sync_for_device()
463 	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;  in nouveau_bo_sync_for_device()
481 	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);  in nouveau_bo_sync_for_cpu()
483 	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;  in nouveau_bo_sync_for_cpu()
504 	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,  in nouveau_bo_validate()
527 		dma_tt = (struct ttm_dma_tt *)nvbo->bo.ttm;  in _nouveau_bo_mem_index()
664 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)  in nouveau_bo_evict_flags()  argument
666 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_evict_flags()
668 	switch (bo->mem.mem_type) {  in nouveau_bo_evict_flags()
695 nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,  in nve0_bo_move_copy()  argument
727 nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,  in nvc0_bo_move_copy()  argument
765 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,  in nvc0_bo_move_m2mf()  argument
804 nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,  in nva3_bo_move_copy()  argument
842 nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,  in nv98_bo_move_exec()  argument
860 nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,  in nv84_bo_move_exec()  argument
894 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,  in nv50_bo_move_m2mf()  argument
981 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,  in nouveau_bo_mem_ctxdma()  argument
990 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,  in nv04_bo_move_m2mf()  argument
1003 	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));  in nv04_bo_move_m2mf()
1004 	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));  in nv04_bo_move_m2mf()
1036 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,  in nouveau_bo_move_prep()  argument
1039 	struct nvkm_mem *old_node = bo->mem.mm_node;  in nouveau_bo_move_prep()
1062 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,  in nouveau_bo_move_m2mf()  argument
1065 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);  in nouveau_bo_move_m2mf()
1076 		ret = nouveau_bo_move_prep(drm, bo, new_mem);  in nouveau_bo_move_m2mf()
1082 	ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, intr);  in nouveau_bo_move_m2mf()
1084 		ret = drm->ttm.move(chan, bo, &bo->mem, new_mem);  in nouveau_bo_move_m2mf()
1088 				ret = ttm_bo_move_accel_cleanup(bo,  in nouveau_bo_move_m2mf()
1162 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,  in nouveau_bo_move_flipd()  argument
1179 	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);  in nouveau_bo_move_flipd()
1183 	ret = ttm_tt_bind(bo->ttm, &tmp_mem);  in nouveau_bo_move_flipd()
1187 	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, &tmp_mem);  in nouveau_bo_move_flipd()
1191 	ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem);  in nouveau_bo_move_flipd()
1193 	ttm_bo_mem_put(bo, &tmp_mem);  in nouveau_bo_move_flipd()
1198 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,  in nouveau_bo_move_flips()  argument
1215 	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);  in nouveau_bo_move_flips()
1219 	ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);  in nouveau_bo_move_flips()
1223 	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, new_mem);  in nouveau_bo_move_flips()
1228 	ttm_bo_mem_put(bo, &tmp_mem);  in nouveau_bo_move_flips()
1233 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)  in nouveau_bo_move_ntfy()  argument
1235 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_move_ntfy()
1239 	if (bo->destroy != nouveau_bo_del_ttm)  in nouveau_bo_move_ntfy()
1254 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,  in nouveau_bo_vm_bind()  argument
1257 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);  in nouveau_bo_vm_bind()
1259 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_vm_bind()
1276 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,  in nouveau_bo_vm_cleanup()  argument
1280 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);  in nouveau_bo_vm_cleanup()
1282 	struct fence *fence = reservation_object_get_excl(bo->resv);  in nouveau_bo_vm_cleanup()
1289 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,  in nouveau_bo_move()  argument
1292 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);  in nouveau_bo_move()
1293 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_move()
1294 	struct ttm_mem_reg *old_mem = &bo->mem;  in nouveau_bo_move()
1302 		ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);  in nouveau_bo_move()
1308 	if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) {  in nouveau_bo_move()
1309 		BUG_ON(bo->mem.mm_node != NULL);  in nouveau_bo_move()
1310 		bo->mem = *new_mem;  in nouveau_bo_move()
1318 			ret = nouveau_bo_move_flipd(bo, evict, intr,  in nouveau_bo_move()
1321 			ret = nouveau_bo_move_flips(bo, evict, intr,  in nouveau_bo_move()
1324 			ret = nouveau_bo_move_m2mf(bo, evict, intr,  in nouveau_bo_move()
1331 	ret = ttm_bo_wait(bo, true, intr, no_wait_gpu);  in nouveau_bo_move()
1333 		ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);  in nouveau_bo_move()
1338 			nouveau_bo_vm_cleanup(bo, NULL, &new_tile);  in nouveau_bo_move()
1340 			nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);  in nouveau_bo_move()
1347 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)  in nouveau_bo_verify_access()  argument
1349 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_bo_verify_access()
1420 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)  in nouveau_ttm_fault_reserve_notify()  argument
1422 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);  in nouveau_ttm_fault_reserve_notify()
1423 	struct nouveau_bo *nvbo = nouveau_bo(bo);  in nouveau_ttm_fault_reserve_notify()
1431 	if (bo->mem.mem_type != TTM_PL_VRAM) {  in nouveau_ttm_fault_reserve_notify()
1436 		if (bo->mem.mem_type == TTM_PL_SYSTEM) {  in nouveau_ttm_fault_reserve_notify()
1448 	    bo->mem.start + bo->mem.num_pages < mappable)  in nouveau_ttm_fault_reserve_notify()
1595 	struct reservation_object *resv = nvbo->bo.resv;  in nouveau_bo_fence()
1634 	const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT;  in nouveau_bo_vma_add()
1642 	if ( nvbo->bo.mem.mem_type != TTM_PL_SYSTEM &&  in nouveau_bo_vma_add()
1643 	    (nvbo->bo.mem.mem_type == TTM_PL_VRAM ||  in nouveau_bo_vma_add()
1645 		nvkm_vm_map(vma, nvbo->bo.mem.mm_node);  in nouveau_bo_vma_add()
1656 		if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM)  in nouveau_bo_vma_del()