1#include <linux/mm.h>
2#include <linux/highmem.h>
3#include <linux/sched.h>
4#include <linux/hugetlb.h>
5
6static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
7			  struct mm_walk *walk)
8{
9	pte_t *pte;
10	int err = 0;
11
12	pte = pte_offset_map(pmd, addr);
13	for (;;) {
14		err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
15		if (err)
16		       break;
17		addr += PAGE_SIZE;
18		if (addr == end)
19			break;
20		pte++;
21	}
22
23	pte_unmap(pte);
24	return err;
25}
26
27static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
28			  struct mm_walk *walk)
29{
30	pmd_t *pmd;
31	unsigned long next;
32	int err = 0;
33
34	pmd = pmd_offset(pud, addr);
35	do {
36again:
37		next = pmd_addr_end(addr, end);
38		if (pmd_none(*pmd) || !walk->vma) {
39			if (walk->pte_hole)
40				err = walk->pte_hole(addr, next, walk);
41			if (err)
42				break;
43			continue;
44		}
45		/*
46		 * This implies that each ->pmd_entry() handler
47		 * needs to know about pmd_trans_huge() pmds
48		 */
49		if (walk->pmd_entry)
50			err = walk->pmd_entry(pmd, addr, next, walk);
51		if (err)
52			break;
53
54		/*
55		 * Check this here so we only break down trans_huge
56		 * pages when we _need_ to
57		 */
58		if (!walk->pte_entry)
59			continue;
60
61		split_huge_page_pmd_mm(walk->mm, addr, pmd);
62		if (pmd_trans_unstable(pmd))
63			goto again;
64		err = walk_pte_range(pmd, addr, next, walk);
65		if (err)
66			break;
67	} while (pmd++, addr = next, addr != end);
68
69	return err;
70}
71
72static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
73			  struct mm_walk *walk)
74{
75	pud_t *pud;
76	unsigned long next;
77	int err = 0;
78
79	pud = pud_offset(pgd, addr);
80	do {
81		next = pud_addr_end(addr, end);
82		if (pud_none_or_clear_bad(pud)) {
83			if (walk->pte_hole)
84				err = walk->pte_hole(addr, next, walk);
85			if (err)
86				break;
87			continue;
88		}
89		if (walk->pmd_entry || walk->pte_entry)
90			err = walk_pmd_range(pud, addr, next, walk);
91		if (err)
92			break;
93	} while (pud++, addr = next, addr != end);
94
95	return err;
96}
97
98static int walk_pgd_range(unsigned long addr, unsigned long end,
99			  struct mm_walk *walk)
100{
101	pgd_t *pgd;
102	unsigned long next;
103	int err = 0;
104
105	pgd = pgd_offset(walk->mm, addr);
106	do {
107		next = pgd_addr_end(addr, end);
108		if (pgd_none_or_clear_bad(pgd)) {
109			if (walk->pte_hole)
110				err = walk->pte_hole(addr, next, walk);
111			if (err)
112				break;
113			continue;
114		}
115		if (walk->pmd_entry || walk->pte_entry)
116			err = walk_pud_range(pgd, addr, next, walk);
117		if (err)
118			break;
119	} while (pgd++, addr = next, addr != end);
120
121	return err;
122}
123
124#ifdef CONFIG_HUGETLB_PAGE
125static unsigned long hugetlb_entry_end(struct hstate *h, unsigned long addr,
126				       unsigned long end)
127{
128	unsigned long boundary = (addr & huge_page_mask(h)) + huge_page_size(h);
129	return boundary < end ? boundary : end;
130}
131
132static int walk_hugetlb_range(unsigned long addr, unsigned long end,
133			      struct mm_walk *walk)
134{
135	struct vm_area_struct *vma = walk->vma;
136	struct hstate *h = hstate_vma(vma);
137	unsigned long next;
138	unsigned long hmask = huge_page_mask(h);
139	pte_t *pte;
140	int err = 0;
141
142	do {
143		next = hugetlb_entry_end(h, addr, end);
144		pte = huge_pte_offset(walk->mm, addr & hmask);
145		if (pte && walk->hugetlb_entry)
146			err = walk->hugetlb_entry(pte, hmask, addr, next, walk);
147		if (err)
148			break;
149	} while (addr = next, addr != end);
150
151	return err;
152}
153
154#else /* CONFIG_HUGETLB_PAGE */
155static int walk_hugetlb_range(unsigned long addr, unsigned long end,
156			      struct mm_walk *walk)
157{
158	return 0;
159}
160
161#endif /* CONFIG_HUGETLB_PAGE */
162
163/*
164 * Decide whether we really walk over the current vma on [@start, @end)
165 * or skip it via the returned value. Return 0 if we do walk over the
166 * current vma, and return 1 if we skip the vma. Negative values means
167 * error, where we abort the current walk.
168 */
169static int walk_page_test(unsigned long start, unsigned long end,
170			struct mm_walk *walk)
171{
172	struct vm_area_struct *vma = walk->vma;
173
174	if (walk->test_walk)
175		return walk->test_walk(start, end, walk);
176
177	/*
178	 * vma(VM_PFNMAP) doesn't have any valid struct pages behind VM_PFNMAP
179	 * range, so we don't walk over it as we do for normal vmas. However,
180	 * Some callers are interested in handling hole range and they don't
181	 * want to just ignore any single address range. Such users certainly
182	 * define their ->pte_hole() callbacks, so let's delegate them to handle
183	 * vma(VM_PFNMAP).
184	 */
185	if (vma->vm_flags & VM_PFNMAP) {
186		int err = 1;
187		if (walk->pte_hole)
188			err = walk->pte_hole(start, end, walk);
189		return err ? err : 1;
190	}
191	return 0;
192}
193
194static int __walk_page_range(unsigned long start, unsigned long end,
195			struct mm_walk *walk)
196{
197	int err = 0;
198	struct vm_area_struct *vma = walk->vma;
199
200	if (vma && is_vm_hugetlb_page(vma)) {
201		if (walk->hugetlb_entry)
202			err = walk_hugetlb_range(start, end, walk);
203	} else
204		err = walk_pgd_range(start, end, walk);
205
206	return err;
207}
208
209/**
210 * walk_page_range - walk page table with caller specific callbacks
211 *
212 * Recursively walk the page table tree of the process represented by @walk->mm
213 * within the virtual address range [@start, @end). During walking, we can do
214 * some caller-specific works for each entry, by setting up pmd_entry(),
215 * pte_entry(), and/or hugetlb_entry(). If you don't set up for some of these
216 * callbacks, the associated entries/pages are just ignored.
217 * The return values of these callbacks are commonly defined like below:
218 *  - 0  : succeeded to handle the current entry, and if you don't reach the
219 *         end address yet, continue to walk.
220 *  - >0 : succeeded to handle the current entry, and return to the caller
221 *         with caller specific value.
222 *  - <0 : failed to handle the current entry, and return to the caller
223 *         with error code.
224 *
225 * Before starting to walk page table, some callers want to check whether
226 * they really want to walk over the current vma, typically by checking
227 * its vm_flags. walk_page_test() and @walk->test_walk() are used for this
228 * purpose.
229 *
230 * struct mm_walk keeps current values of some common data like vma and pmd,
231 * which are useful for the access from callbacks. If you want to pass some
232 * caller-specific data to callbacks, @walk->private should be helpful.
233 *
234 * Locking:
235 *   Callers of walk_page_range() and walk_page_vma() should hold
236 *   @walk->mm->mmap_sem, because these function traverse vma list and/or
237 *   access to vma's data.
238 */
239int walk_page_range(unsigned long start, unsigned long end,
240		    struct mm_walk *walk)
241{
242	int err = 0;
243	unsigned long next;
244	struct vm_area_struct *vma;
245
246	if (start >= end)
247		return -EINVAL;
248
249	if (!walk->mm)
250		return -EINVAL;
251
252	VM_BUG_ON_MM(!rwsem_is_locked(&walk->mm->mmap_sem), walk->mm);
253
254	vma = find_vma(walk->mm, start);
255	do {
256		if (!vma) { /* after the last vma */
257			walk->vma = NULL;
258			next = end;
259		} else if (start < vma->vm_start) { /* outside vma */
260			walk->vma = NULL;
261			next = min(end, vma->vm_start);
262		} else { /* inside vma */
263			walk->vma = vma;
264			next = min(end, vma->vm_end);
265			vma = vma->vm_next;
266
267			err = walk_page_test(start, next, walk);
268			if (err > 0) {
269				/*
270				 * positive return values are purely for
271				 * controlling the pagewalk, so should never
272				 * be passed to the callers.
273				 */
274				err = 0;
275				continue;
276			}
277			if (err < 0)
278				break;
279		}
280		if (walk->vma || walk->pte_hole)
281			err = __walk_page_range(start, next, walk);
282		if (err)
283			break;
284	} while (start = next, start < end);
285	return err;
286}
287
288int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk)
289{
290	int err;
291
292	if (!walk->mm)
293		return -EINVAL;
294
295	VM_BUG_ON(!rwsem_is_locked(&walk->mm->mmap_sem));
296	VM_BUG_ON(!vma);
297	walk->vma = vma;
298	err = walk_page_test(vma->vm_start, vma->vm_end, walk);
299	if (err > 0)
300		return 0;
301	if (err < 0)
302		return err;
303	return __walk_page_range(vma->vm_start, vma->vm_end, walk);
304}
305