51 static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)  in ttm_tt_alloc_page_directory()  argument
53 	ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*));  in ttm_tt_alloc_page_directory()
56 static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm)  in ttm_dma_tt_alloc_page_directory()  argument
58 	ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages,  in ttm_dma_tt_alloc_page_directory()
59 					  sizeof(*ttm->ttm.pages) +  in ttm_dma_tt_alloc_page_directory()
60 					  sizeof(*ttm->dma_address) +  in ttm_dma_tt_alloc_page_directory()
61 					  sizeof(*ttm->cpu_address));  in ttm_dma_tt_alloc_page_directory()
62 	ttm->cpu_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages);  in ttm_dma_tt_alloc_page_directory()
63 	ttm->dma_address = (void *) (ttm->cpu_address + ttm->ttm.num_pages);  in ttm_dma_tt_alloc_page_directory()
106 static int ttm_tt_set_caching(struct ttm_tt *ttm,  in ttm_tt_set_caching()  argument
113 	if (ttm->caching_state == c_state)  in ttm_tt_set_caching()
116 	if (ttm->state == tt_unpopulated) {  in ttm_tt_set_caching()
118 		ttm->caching_state = c_state;  in ttm_tt_set_caching()
122 	if (ttm->caching_state == tt_cached)  in ttm_tt_set_caching()
123 		drm_clflush_pages(ttm->pages, ttm->num_pages);  in ttm_tt_set_caching()
125 	for (i = 0; i < ttm->num_pages; ++i) {  in ttm_tt_set_caching()
126 		cur_page = ttm->pages[i];  in ttm_tt_set_caching()
129 						      ttm->caching_state,  in ttm_tt_set_caching()
136 	ttm->caching_state = c_state;  in ttm_tt_set_caching()
142 		cur_page = ttm->pages[j];  in ttm_tt_set_caching()
145 						      ttm->caching_state);  in ttm_tt_set_caching()
152 int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)  in ttm_tt_set_placement_caching()  argument
163 	return ttm_tt_set_caching(ttm, state);  in ttm_tt_set_placement_caching()
167 void ttm_tt_destroy(struct ttm_tt *ttm)  in ttm_tt_destroy()  argument
169 	if (unlikely(ttm == NULL))  in ttm_tt_destroy()
172 	if (ttm->state == tt_bound) {  in ttm_tt_destroy()
173 		ttm_tt_unbind(ttm);  in ttm_tt_destroy()
176 	if (ttm->state == tt_unbound)  in ttm_tt_destroy()
177 		ttm_tt_unpopulate(ttm);  in ttm_tt_destroy()
179 	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&  in ttm_tt_destroy()
180 	    ttm->swap_storage)  in ttm_tt_destroy()
181 		fput(ttm->swap_storage);  in ttm_tt_destroy()
183 	ttm->swap_storage = NULL;  in ttm_tt_destroy()
184 	ttm->func->destroy(ttm);  in ttm_tt_destroy()
187 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev,  in ttm_tt_init()  argument
191 	ttm->bdev = bdev;  in ttm_tt_init()
192 	ttm->glob = bdev->glob;  in ttm_tt_init()
193 	ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;  in ttm_tt_init()
194 	ttm->caching_state = tt_cached;  in ttm_tt_init()
195 	ttm->page_flags = page_flags;  in ttm_tt_init()
196 	ttm->dummy_read_page = dummy_read_page;  in ttm_tt_init()
197 	ttm->state = tt_unpopulated;  in ttm_tt_init()
198 	ttm->swap_storage = NULL;  in ttm_tt_init()
200 	ttm_tt_alloc_page_directory(ttm);  in ttm_tt_init()
201 	if (!ttm->pages) {  in ttm_tt_init()
202 		ttm_tt_destroy(ttm);  in ttm_tt_init()
210 void ttm_tt_fini(struct ttm_tt *ttm)  in ttm_tt_fini()  argument
212 	drm_free_large(ttm->pages);  in ttm_tt_fini()
213 	ttm->pages = NULL;  in ttm_tt_fini()
221 	struct ttm_tt *ttm = &ttm_dma->ttm;  in ttm_dma_tt_init()  local
223 	ttm->bdev = bdev;  in ttm_dma_tt_init()
224 	ttm->glob = bdev->glob;  in ttm_dma_tt_init()
225 	ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;  in ttm_dma_tt_init()
226 	ttm->caching_state = tt_cached;  in ttm_dma_tt_init()
227 	ttm->page_flags = page_flags;  in ttm_dma_tt_init()
228 	ttm->dummy_read_page = dummy_read_page;  in ttm_dma_tt_init()
229 	ttm->state = tt_unpopulated;  in ttm_dma_tt_init()
230 	ttm->swap_storage = NULL;  in ttm_dma_tt_init()
234 	if (!ttm->pages) {  in ttm_dma_tt_init()
235 		ttm_tt_destroy(ttm);  in ttm_dma_tt_init()
245 	struct ttm_tt *ttm = &ttm_dma->ttm;  in ttm_dma_tt_fini()  local
247 	drm_free_large(ttm->pages);  in ttm_dma_tt_fini()
248 	ttm->pages = NULL;  in ttm_dma_tt_fini()
254 void ttm_tt_unbind(struct ttm_tt *ttm)  in ttm_tt_unbind()  argument
258 	if (ttm->state == tt_bound) {  in ttm_tt_unbind()
259 		ret = ttm->func->unbind(ttm);  in ttm_tt_unbind()
261 		ttm->state = tt_unbound;  in ttm_tt_unbind()
265 int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)  in ttm_tt_bind()  argument
269 	if (!ttm)  in ttm_tt_bind()
272 	if (ttm->state == tt_bound)  in ttm_tt_bind()
275 	ret = ttm->bdev->driver->ttm_tt_populate(ttm);  in ttm_tt_bind()
279 	ret = ttm->func->bind(ttm, bo_mem);  in ttm_tt_bind()
283 	ttm->state = tt_bound;  in ttm_tt_bind()
289 int ttm_tt_swapin(struct ttm_tt *ttm)  in ttm_tt_swapin()  argument
298 	swap_storage = ttm->swap_storage;  in ttm_tt_swapin()
303 	for (i = 0; i < ttm->num_pages; ++i) {  in ttm_tt_swapin()
309 		to_page = ttm->pages[i];  in ttm_tt_swapin()
317 	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))  in ttm_tt_swapin()
319 	ttm->swap_storage = NULL;  in ttm_tt_swapin()
320 	ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;  in ttm_tt_swapin()
327 int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)  in ttm_tt_swapout()  argument
336 	BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);  in ttm_tt_swapout()
337 	BUG_ON(ttm->caching_state != tt_cached);  in ttm_tt_swapout()
341 						ttm->num_pages << PAGE_SHIFT,  in ttm_tt_swapout()
352 	for (i = 0; i < ttm->num_pages; ++i) {  in ttm_tt_swapout()
353 		from_page = ttm->pages[i];  in ttm_tt_swapout()
367 	ttm_tt_unpopulate(ttm);  in ttm_tt_swapout()
368 	ttm->swap_storage = swap_storage;  in ttm_tt_swapout()
369 	ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;  in ttm_tt_swapout()
371 		ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;  in ttm_tt_swapout()
381 static void ttm_tt_clear_mapping(struct ttm_tt *ttm)  in ttm_tt_clear_mapping()  argument
384 	struct page **page = ttm->pages;  in ttm_tt_clear_mapping()
386 	if (ttm->page_flags & TTM_PAGE_FLAG_SG)  in ttm_tt_clear_mapping()
389 	for (i = 0; i < ttm->num_pages; ++i) {  in ttm_tt_clear_mapping()
395 void ttm_tt_unpopulate(struct ttm_tt *ttm)  in ttm_tt_unpopulate()  argument
397 	if (ttm->state == tt_unpopulated)  in ttm_tt_unpopulate()
400 	ttm_tt_clear_mapping(ttm);  in ttm_tt_unpopulate()
401 	ttm->bdev->driver->ttm_tt_unpopulate(ttm);  in ttm_tt_unpopulate()