1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
30
31#define pr_fmt(fmt) "[TTM] " fmt
32
33#include <linux/sched.h>
34#include <linux/highmem.h>
35#include <linux/pagemap.h>
36#include <linux/shmem_fs.h>
37#include <linux/file.h>
38#include <linux/swap.h>
39#include <linux/slab.h>
40#include <linux/export.h>
41#include <drm/drm_cache.h>
42#include <drm/drm_mem_util.h>
43#include <drm/ttm/ttm_module.h>
44#include <drm/ttm/ttm_bo_driver.h>
45#include <drm/ttm/ttm_placement.h>
46#include <drm/ttm/ttm_page_alloc.h>
47
48/**
49 * Allocates storage for pointers to the pages that back the ttm.
50 */
51static void ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
52{
53	ttm->pages = drm_calloc_large(ttm->num_pages, sizeof(void*));
54}
55
56static void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
57{
58	ttm->ttm.pages = drm_calloc_large(ttm->ttm.num_pages,
59					  sizeof(*ttm->ttm.pages) +
60					  sizeof(*ttm->dma_address) +
61					  sizeof(*ttm->cpu_address));
62	ttm->cpu_address = (void *) (ttm->ttm.pages + ttm->ttm.num_pages);
63	ttm->dma_address = (void *) (ttm->cpu_address + ttm->ttm.num_pages);
64}
65
66#ifdef CONFIG_X86
67static inline int ttm_tt_set_page_caching(struct page *p,
68					  enum ttm_caching_state c_old,
69					  enum ttm_caching_state c_new)
70{
71	int ret = 0;
72
73	if (PageHighMem(p))
74		return 0;
75
76	if (c_old != tt_cached) {
77		/* p isn't in the default caching state, set it to
78		 * writeback first to free its current memtype. */
79
80		ret = set_pages_wb(p, 1);
81		if (ret)
82			return ret;
83	}
84
85	if (c_new == tt_wc)
86		ret = set_memory_wc((unsigned long) page_address(p), 1);
87	else if (c_new == tt_uncached)
88		ret = set_pages_uc(p, 1);
89
90	return ret;
91}
92#else /* CONFIG_X86 */
93static inline int ttm_tt_set_page_caching(struct page *p,
94					  enum ttm_caching_state c_old,
95					  enum ttm_caching_state c_new)
96{
97	return 0;
98}
99#endif /* CONFIG_X86 */
100
101/*
102 * Change caching policy for the linear kernel map
103 * for range of pages in a ttm.
104 */
105
106static int ttm_tt_set_caching(struct ttm_tt *ttm,
107			      enum ttm_caching_state c_state)
108{
109	int i, j;
110	struct page *cur_page;
111	int ret;
112
113	if (ttm->caching_state == c_state)
114		return 0;
115
116	if (ttm->state == tt_unpopulated) {
117		/* Change caching but don't populate */
118		ttm->caching_state = c_state;
119		return 0;
120	}
121
122	if (ttm->caching_state == tt_cached)
123		drm_clflush_pages(ttm->pages, ttm->num_pages);
124
125	for (i = 0; i < ttm->num_pages; ++i) {
126		cur_page = ttm->pages[i];
127		if (likely(cur_page != NULL)) {
128			ret = ttm_tt_set_page_caching(cur_page,
129						      ttm->caching_state,
130						      c_state);
131			if (unlikely(ret != 0))
132				goto out_err;
133		}
134	}
135
136	ttm->caching_state = c_state;
137
138	return 0;
139
140out_err:
141	for (j = 0; j < i; ++j) {
142		cur_page = ttm->pages[j];
143		if (likely(cur_page != NULL)) {
144			(void)ttm_tt_set_page_caching(cur_page, c_state,
145						      ttm->caching_state);
146		}
147	}
148
149	return ret;
150}
151
152int ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement)
153{
154	enum ttm_caching_state state;
155
156	if (placement & TTM_PL_FLAG_WC)
157		state = tt_wc;
158	else if (placement & TTM_PL_FLAG_UNCACHED)
159		state = tt_uncached;
160	else
161		state = tt_cached;
162
163	return ttm_tt_set_caching(ttm, state);
164}
165EXPORT_SYMBOL(ttm_tt_set_placement_caching);
166
167void ttm_tt_destroy(struct ttm_tt *ttm)
168{
169	if (unlikely(ttm == NULL))
170		return;
171
172	if (ttm->state == tt_bound) {
173		ttm_tt_unbind(ttm);
174	}
175
176	if (ttm->state == tt_unbound)
177		ttm_tt_unpopulate(ttm);
178
179	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) &&
180	    ttm->swap_storage)
181		fput(ttm->swap_storage);
182
183	ttm->swap_storage = NULL;
184	ttm->func->destroy(ttm);
185}
186
187int ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev,
188		unsigned long size, uint32_t page_flags,
189		struct page *dummy_read_page)
190{
191	ttm->bdev = bdev;
192	ttm->glob = bdev->glob;
193	ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
194	ttm->caching_state = tt_cached;
195	ttm->page_flags = page_flags;
196	ttm->dummy_read_page = dummy_read_page;
197	ttm->state = tt_unpopulated;
198	ttm->swap_storage = NULL;
199
200	ttm_tt_alloc_page_directory(ttm);
201	if (!ttm->pages) {
202		ttm_tt_destroy(ttm);
203		pr_err("Failed allocating page table\n");
204		return -ENOMEM;
205	}
206	return 0;
207}
208EXPORT_SYMBOL(ttm_tt_init);
209
210void ttm_tt_fini(struct ttm_tt *ttm)
211{
212	drm_free_large(ttm->pages);
213	ttm->pages = NULL;
214}
215EXPORT_SYMBOL(ttm_tt_fini);
216
217int ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev,
218		unsigned long size, uint32_t page_flags,
219		struct page *dummy_read_page)
220{
221	struct ttm_tt *ttm = &ttm_dma->ttm;
222
223	ttm->bdev = bdev;
224	ttm->glob = bdev->glob;
225	ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
226	ttm->caching_state = tt_cached;
227	ttm->page_flags = page_flags;
228	ttm->dummy_read_page = dummy_read_page;
229	ttm->state = tt_unpopulated;
230	ttm->swap_storage = NULL;
231
232	INIT_LIST_HEAD(&ttm_dma->pages_list);
233	ttm_dma_tt_alloc_page_directory(ttm_dma);
234	if (!ttm->pages) {
235		ttm_tt_destroy(ttm);
236		pr_err("Failed allocating page table\n");
237		return -ENOMEM;
238	}
239	return 0;
240}
241EXPORT_SYMBOL(ttm_dma_tt_init);
242
243void ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma)
244{
245	struct ttm_tt *ttm = &ttm_dma->ttm;
246
247	drm_free_large(ttm->pages);
248	ttm->pages = NULL;
249	ttm_dma->cpu_address = NULL;
250	ttm_dma->dma_address = NULL;
251}
252EXPORT_SYMBOL(ttm_dma_tt_fini);
253
254void ttm_tt_unbind(struct ttm_tt *ttm)
255{
256	int ret;
257
258	if (ttm->state == tt_bound) {
259		ret = ttm->func->unbind(ttm);
260		BUG_ON(ret);
261		ttm->state = tt_unbound;
262	}
263}
264
265int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
266{
267	int ret = 0;
268
269	if (!ttm)
270		return -EINVAL;
271
272	if (ttm->state == tt_bound)
273		return 0;
274
275	ret = ttm->bdev->driver->ttm_tt_populate(ttm);
276	if (ret)
277		return ret;
278
279	ret = ttm->func->bind(ttm, bo_mem);
280	if (unlikely(ret != 0))
281		return ret;
282
283	ttm->state = tt_bound;
284
285	return 0;
286}
287EXPORT_SYMBOL(ttm_tt_bind);
288
289int ttm_tt_swapin(struct ttm_tt *ttm)
290{
291	struct address_space *swap_space;
292	struct file *swap_storage;
293	struct page *from_page;
294	struct page *to_page;
295	int i;
296	int ret = -ENOMEM;
297
298	swap_storage = ttm->swap_storage;
299	BUG_ON(swap_storage == NULL);
300
301	swap_space = file_inode(swap_storage)->i_mapping;
302
303	for (i = 0; i < ttm->num_pages; ++i) {
304		from_page = shmem_read_mapping_page(swap_space, i);
305		if (IS_ERR(from_page)) {
306			ret = PTR_ERR(from_page);
307			goto out_err;
308		}
309		to_page = ttm->pages[i];
310		if (unlikely(to_page == NULL))
311			goto out_err;
312
313		copy_highpage(to_page, from_page);
314		page_cache_release(from_page);
315	}
316
317	if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP))
318		fput(swap_storage);
319	ttm->swap_storage = NULL;
320	ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED;
321
322	return 0;
323out_err:
324	return ret;
325}
326
327int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistent_swap_storage)
328{
329	struct address_space *swap_space;
330	struct file *swap_storage;
331	struct page *from_page;
332	struct page *to_page;
333	int i;
334	int ret = -ENOMEM;
335
336	BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);
337	BUG_ON(ttm->caching_state != tt_cached);
338
339	if (!persistent_swap_storage) {
340		swap_storage = shmem_file_setup("ttm swap",
341						ttm->num_pages << PAGE_SHIFT,
342						0);
343		if (unlikely(IS_ERR(swap_storage))) {
344			pr_err("Failed allocating swap storage\n");
345			return PTR_ERR(swap_storage);
346		}
347	} else
348		swap_storage = persistent_swap_storage;
349
350	swap_space = file_inode(swap_storage)->i_mapping;
351
352	for (i = 0; i < ttm->num_pages; ++i) {
353		from_page = ttm->pages[i];
354		if (unlikely(from_page == NULL))
355			continue;
356		to_page = shmem_read_mapping_page(swap_space, i);
357		if (unlikely(IS_ERR(to_page))) {
358			ret = PTR_ERR(to_page);
359			goto out_err;
360		}
361		copy_highpage(to_page, from_page);
362		set_page_dirty(to_page);
363		mark_page_accessed(to_page);
364		page_cache_release(to_page);
365	}
366
367	ttm_tt_unpopulate(ttm);
368	ttm->swap_storage = swap_storage;
369	ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
370	if (persistent_swap_storage)
371		ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP;
372
373	return 0;
374out_err:
375	if (!persistent_swap_storage)
376		fput(swap_storage);
377
378	return ret;
379}
380
381static void ttm_tt_clear_mapping(struct ttm_tt *ttm)
382{
383	pgoff_t i;
384	struct page **page = ttm->pages;
385
386	if (ttm->page_flags & TTM_PAGE_FLAG_SG)
387		return;
388
389	for (i = 0; i < ttm->num_pages; ++i) {
390		(*page)->mapping = NULL;
391		(*page++)->index = 0;
392	}
393}
394
395void ttm_tt_unpopulate(struct ttm_tt *ttm)
396{
397	if (ttm->state == tt_unpopulated)
398		return;
399
400	ttm_tt_clear_mapping(ttm);
401	ttm->bdev->driver->ttm_tt_unpopulate(ttm);
402}
403