74 static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,  in gup_pte_range()  argument
96 	ptep = pte_offset_map(&pmd, addr);  in gup_pte_range()
108 		__flush_anon_page(page, addr);  in gup_pte_range()
113 	} while (ptep++, addr += PAGE_SIZE, addr != end);  in gup_pte_range()
119 static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,  in gup_pmd_range()  argument
125 	pmdp = pmd_offset(&pud, addr);  in gup_pmd_range()
129 		next = pmd_addr_end(addr, end);  in gup_pmd_range()
132 		if (!gup_pte_range(pmd, addr, next, write, pages, nr))  in gup_pmd_range()
134 	} while (pmdp++, addr = next, addr != end);  in gup_pmd_range()
139 static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,  in gup_pud_range()  argument
145 	pudp = pud_offset(&pgd, addr);  in gup_pud_range()
149 		next = pud_addr_end(addr, end);  in gup_pud_range()
152 		if (!gup_pmd_range(pud, addr, next, write, pages, nr))  in gup_pud_range()
154 	} while (pudp++, addr = next, addr != end);  in gup_pud_range()
167 	unsigned long addr, len, end;  in __get_user_pages_fast()  local
174 	addr = start;  in __get_user_pages_fast()
186 	pgdp = pgd_offset(mm, addr);  in __get_user_pages_fast()
190 		next = pgd_addr_end(addr, end);  in __get_user_pages_fast()
193 		if (!gup_pud_range(pgd, addr, next, write, pages, &nr))  in __get_user_pages_fast()
195 	} while (pgdp++, addr = next, addr != end);  in __get_user_pages_fast()
221 	unsigned long addr, len, end;  in get_user_pages_fast()  local
227 	addr = start;  in get_user_pages_fast()
235 	pgdp = pgd_offset(mm, addr);  in get_user_pages_fast()
239 		next = pgd_addr_end(addr, end);  in get_user_pages_fast()
242 		if (!gup_pud_range(pgd, addr, next, write, pages, &nr))  in get_user_pages_fast()
244 	} while (pgdp++, addr = next, addr != end);  in get_user_pages_fast()