37 static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,  in gup_pte_range()  argument
40 	pte_t *ptep = pte_offset_map(&pmd, addr);  in gup_pte_range()
57 	} while (ptep++, addr += PAGE_SIZE, addr != end);  in gup_pte_range()
71 static int gup_huge_pmd(pmd_t pmd, unsigned long addr, unsigned long end,  in gup_huge_pmd()  argument
86 	page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);  in gup_huge_pmd()
95 	} while (addr += PAGE_SIZE, addr != end);  in gup_huge_pmd()
101 static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,  in gup_pmd_range()  argument
107 	pmdp = pmd_offset(&pud, addr);  in gup_pmd_range()
111 		next = pmd_addr_end(addr, end);  in gup_pmd_range()
126 			if (!gup_huge_pmd(pmd, addr, next, write, pages,nr))  in gup_pmd_range()
129 			if (!gup_pte_range(pmd, addr, next, write, pages,nr))  in gup_pmd_range()
132 	} while (pmdp++, addr = next, addr != end);  in gup_pmd_range()
137 static int gup_huge_pud(pud_t pud, unsigned long addr, unsigned long end,  in gup_huge_pud()  argument
152 	page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);  in gup_huge_pud()
161 	} while (addr += PAGE_SIZE, addr != end);  in gup_huge_pud()
167 static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,  in gup_pud_range()  argument
173 	pudp = pud_offset(&pgd, addr);  in gup_pud_range()
177 		next = pud_addr_end(addr, end);  in gup_pud_range()
181 			if (!gup_huge_pud(pud, addr, next, write, pages,nr))  in gup_pud_range()
184 			if (!gup_pmd_range(pud, addr, next, write, pages,nr))  in gup_pud_range()
187 	} while (pudp++, addr = next, addr != end);  in gup_pud_range()
200 	unsigned long addr, len, end;  in __get_user_pages_fast()  local
207 	addr = start;  in __get_user_pages_fast()
232 	pgdp = pgd_offset(mm, addr);  in __get_user_pages_fast()
236 		next = pgd_addr_end(addr, end);  in __get_user_pages_fast()
239 		if (!gup_pud_range(pgd, addr, next, write, pages, &nr))  in __get_user_pages_fast()
241 	} while (pgdp++, addr = next, addr != end);  in __get_user_pages_fast()
267 	unsigned long addr, len, end;  in get_user_pages_fast()  local
273 	addr = start;  in get_user_pages_fast()
282 	pgdp = pgd_offset(mm, addr);  in get_user_pages_fast()
286 		next = pgd_addr_end(addr, end);  in get_user_pages_fast()
289 		if (!gup_pud_range(pgd, addr, next, write, pages, &nr))  in get_user_pages_fast()
291 	} while (pgdp++, addr = next, addr != end);  in get_user_pages_fast()