91 	       bo, bo->mem.num_pages, bo->mem.size >> 10,  in ttm_bo_mem_space_debug()
92 	       bo->mem.size >> 20);  in ttm_bo_mem_space_debug()
146 	BUG_ON(bo->mem.mm_node != NULL);  in ttm_bo_release_list()
171 	if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {  in ttm_bo_add_to_lru()
175 		man = &bdev->man[bo->mem.mem_type];  in ttm_bo_add_to_lru()
277 				  struct ttm_mem_reg *mem,  in ttm_bo_handle_move_mem()  argument
282 	bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);  in ttm_bo_handle_move_mem()
283 	bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);  in ttm_bo_handle_move_mem()
284 	struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
285 	struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];  in ttm_bo_handle_move_mem()
289 	    ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {  in ttm_bo_handle_move_mem()
309 		ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);  in ttm_bo_handle_move_mem()
313 		if (mem->mem_type != TTM_PL_SYSTEM) {  in ttm_bo_handle_move_mem()
314 			ret = ttm_tt_bind(bo->ttm, mem);  in ttm_bo_handle_move_mem()
319 		if (bo->mem.mem_type == TTM_PL_SYSTEM) {  in ttm_bo_handle_move_mem()
321 				bdev->driver->move_notify(bo, mem);  in ttm_bo_handle_move_mem()
322 			bo->mem = *mem;  in ttm_bo_handle_move_mem()
323 			mem->mm_node = NULL;  in ttm_bo_handle_move_mem()
329 		bdev->driver->move_notify(bo, mem);  in ttm_bo_handle_move_mem()
333 		ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem);  in ttm_bo_handle_move_mem()
336 					 no_wait_gpu, mem);  in ttm_bo_handle_move_mem()
338 		ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem);  in ttm_bo_handle_move_mem()
342 			struct ttm_mem_reg tmp_mem = *mem;  in ttm_bo_handle_move_mem()
343 			*mem = bo->mem;  in ttm_bo_handle_move_mem()
344 			bo->mem = tmp_mem;  in ttm_bo_handle_move_mem()
345 			bdev->driver->move_notify(bo, mem);  in ttm_bo_handle_move_mem()
346 			bo->mem = *mem;  in ttm_bo_handle_move_mem()
347 			*mem = tmp_mem;  in ttm_bo_handle_move_mem()
356 			ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);  in ttm_bo_handle_move_mem()
363 	if (bo->mem.mm_node) {  in ttm_bo_handle_move_mem()
364 		bo->offset = (bo->mem.start << PAGE_SHIFT) +  in ttm_bo_handle_move_mem()
365 		    bdev->man[bo->mem.mem_type].gpu_offset;  in ttm_bo_handle_move_mem()
366 		bo->cur_placement = bo->mem.placement;  in ttm_bo_handle_move_mem()
373 	new_man = &bdev->man[bo->mem.mem_type];  in ttm_bo_handle_move_mem()
401 	ttm_bo_mem_put(bo, &bo->mem);  in ttm_bo_cleanup_memtype_use()
454 		if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {  in ttm_bo_cleanup_refs_or_queue()
455 			bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;  in ttm_bo_cleanup_refs_or_queue()
626 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_release()
678 	evict_mem = bo->mem;  in ttm_bo_evict()
729 				if (place->fpfn >= (bo->mem.start + bo->mem.size) ||  in ttm_mem_evict_first()
730 				    (place->lpfn && place->lpfn <= bo->mem.start)) {  in ttm_mem_evict_first()
769 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)  in ttm_bo_mem_put()  argument
771 	struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];  in ttm_bo_mem_put()
773 	if (mem->mm_node)  in ttm_bo_mem_put()
774 		(*man->func->put_node)(man, mem);  in ttm_bo_mem_put()
785 					struct ttm_mem_reg *mem,  in ttm_bo_mem_force_space()  argument
794 		ret = (*man->func->get_node)(man, bo, place, mem);  in ttm_bo_mem_force_space()
797 		if (mem->mm_node)  in ttm_bo_mem_force_space()
804 	if (mem->mm_node == NULL)  in ttm_bo_mem_force_space()
806 	mem->mem_type = mem_type;  in ttm_bo_mem_force_space()
864 			struct ttm_mem_reg *mem,  in ttm_bo_mem_space()  argument
877 	mem->mm_node = NULL;  in ttm_bo_mem_space()
895 		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,  in ttm_bo_mem_space()
907 		ret = (*man->func->get_node)(man, bo, place, mem);  in ttm_bo_mem_space()
911 		if (mem->mm_node)  in ttm_bo_mem_space()
915 	if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {  in ttm_bo_mem_space()
916 		mem->mem_type = mem_type;  in ttm_bo_mem_space()
917 		mem->placement = cur_flags;  in ttm_bo_mem_space()
934 		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,  in ttm_bo_mem_space()
944 			mem->mem_type = mem_type;  in ttm_bo_mem_space()
945 			mem->placement = cur_flags;  in ttm_bo_mem_space()
946 			mem->mm_node = NULL;  in ttm_bo_mem_space()
950 		ret = ttm_bo_mem_force_space(bo, mem_type, place, mem,  in ttm_bo_mem_space()
952 		if (ret == 0 && mem->mm_node) {  in ttm_bo_mem_space()
953 			mem->placement = cur_flags;  in ttm_bo_mem_space()
975 	struct ttm_mem_reg mem;  in ttm_bo_move_buffer()  local
987 	mem.num_pages = bo->num_pages;  in ttm_bo_move_buffer()
988 	mem.size = mem.num_pages << PAGE_SHIFT;  in ttm_bo_move_buffer()
989 	mem.page_alignment = bo->mem.page_alignment;  in ttm_bo_move_buffer()
990 	mem.bus.io_reserved_vm = false;  in ttm_bo_move_buffer()
991 	mem.bus.io_reserved_count = 0;  in ttm_bo_move_buffer()
995 	ret = ttm_bo_mem_space(bo, placement, &mem,  in ttm_bo_move_buffer()
999 	ret = ttm_bo_handle_move_mem(bo, &mem, false,  in ttm_bo_move_buffer()
1002 	if (ret && mem.mm_node)  in ttm_bo_move_buffer()
1003 		ttm_bo_mem_put(bo, &mem);  in ttm_bo_move_buffer()
1008 			      struct ttm_mem_reg *mem,  in ttm_bo_mem_compat()  argument
1015 		if (mem->mm_node &&  in ttm_bo_mem_compat()
1016 		    (mem->start < heap->fpfn ||  in ttm_bo_mem_compat()
1017 		     (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))  in ttm_bo_mem_compat()
1021 		if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&  in ttm_bo_mem_compat()
1022 		    (*new_flags & mem->placement & TTM_PL_MASK_MEM))  in ttm_bo_mem_compat()
1028 		if (mem->mm_node &&  in ttm_bo_mem_compat()
1029 		    (mem->start < heap->fpfn ||  in ttm_bo_mem_compat()
1030 		     (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))  in ttm_bo_mem_compat()
1034 		if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&  in ttm_bo_mem_compat()
1035 		    (*new_flags & mem->placement & TTM_PL_MASK_MEM))  in ttm_bo_mem_compat()
1054 	if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {  in ttm_bo_validate()
1064 		ttm_flag_masked(&bo->mem.placement, new_flags,  in ttm_bo_validate()
1070 	if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {  in ttm_bo_validate()
1131 	bo->mem.size = num_pages << PAGE_SHIFT;  in ttm_bo_init()
1132 	bo->mem.mem_type = TTM_PL_SYSTEM;  in ttm_bo_init()
1133 	bo->mem.num_pages = bo->num_pages;  in ttm_bo_init()
1134 	bo->mem.mm_node = NULL;  in ttm_bo_init()
1135 	bo->mem.page_alignment = page_alignment;  in ttm_bo_init()
1136 	bo->mem.bus.io_reserved_vm = false;  in ttm_bo_init()
1137 	bo->mem.bus.io_reserved_count = 0;  in ttm_bo_init()
1139 	bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);  in ttm_bo_init()
1160 					 bo->mem.num_pages);  in ttm_bo_init()
1500 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)  in ttm_mem_reg_is_pci()  argument
1502 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];  in ttm_mem_reg_is_pci()
1505 		if (mem->mem_type == TTM_PL_SYSTEM)  in ttm_mem_reg_is_pci()
1511 		if (mem->placement & TTM_PL_FLAG_CACHED)  in ttm_mem_reg_is_pci()
1528 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];  in ttm_bo_unmap_virtual()
1659 	if ((bo->mem.placement & swap_placement) != swap_placement) {  in ttm_bo_swapout()
1662 		evict_mem = bo->mem;  in ttm_bo_swapout()