21 static struct page *no_page_table(struct vm_area_struct *vma,  in no_page_table()
61 static struct page *follow_page_pte(struct vm_area_struct *vma,  in follow_page_pte()
65 	struct page *page;  in follow_page_pte()  local
100 	page = vm_normal_page(vma, address, pte);  in follow_page_pte()
101 	if (unlikely(!page)) {  in follow_page_pte()
104 			page = ERR_PTR(-EFAULT);  in follow_page_pte()
109 			page = pte_page(pte);  in follow_page_pte()
114 			page = ERR_PTR(ret);  in follow_page_pte()
120 		get_page_foll(page);  in follow_page_pte()
123 		    !pte_dirty(pte) && !PageDirty(page))  in follow_page_pte()
124 			set_page_dirty(page);  in follow_page_pte()
130 		mark_page_accessed(page);  in follow_page_pte()
142 		if (page->mapping && trylock_page(page)) {  in follow_page_pte()
150 			mlock_vma_page(page);  in follow_page_pte()
151 			unlock_page(page);  in follow_page_pte()
156 	return page;  in follow_page_pte()
177 struct page *follow_page_mask(struct vm_area_struct *vma,  in follow_page_mask()
185 	struct page *page;  in follow_page_mask()  local
190 	page = follow_huge_addr(mm, address, flags & FOLL_WRITE);  in follow_page_mask()
191 	if (!IS_ERR(page)) {  in follow_page_mask()
193 		return page;  in follow_page_mask()
204 		page = follow_huge_pud(mm, address, pud, flags);  in follow_page_mask()
205 		if (page)  in follow_page_mask()
206 			return page;  in follow_page_mask()
216 		page = follow_huge_pmd(mm, address, pmd, flags);  in follow_page_mask()
217 		if (page)  in follow_page_mask()
218 			return page;  in follow_page_mask()
234 				page = follow_trans_huge_pmd(vma, address,  in follow_page_mask()
238 				return page;  in follow_page_mask()
248 		struct page **page)  in get_gate_page()  argument
274 	if (!page)  in get_gate_page()
276 	*page = vm_normal_page(*vma, address, *pte);  in get_gate_page()
277 	if (!*page) {  in get_gate_page()
280 		*page = pte_page(*pte);  in get_gate_page()
282 	get_page(*page);  in get_gate_page()
455 		unsigned int gup_flags, struct page **pages,  in __get_user_pages()
476 		struct page *page;  in __get_user_pages()  local
511 		page = follow_page_mask(vma, start, foll_flags, &page_mask);  in __get_user_pages()
512 		if (!page) {  in __get_user_pages()
529 		} else if (PTR_ERR(page) == -EEXIST) {  in __get_user_pages()
535 		} else if (IS_ERR(page)) {  in __get_user_pages()
536 			return i ? i : PTR_ERR(page);  in __get_user_pages()
539 			pages[i] = page;  in __get_user_pages()
540 			flush_anon_page(vma, page, start);  in __get_user_pages()
541 			flush_dcache_page(page);  in __get_user_pages()
626 						struct page **pages,  in __get_user_pages_locked()
740 			   int write, int force, struct page **pages,  in get_user_pages_locked()
760 					       int write, int force, struct page **pages,  in __get_user_pages_unlocked()
793 			     int write, int force, struct page **pages)  in get_user_pages_unlocked()
857 		int force, struct page **pages, struct vm_area_struct **vmas)  in get_user_pages()
1000 struct page *get_dump_page(unsigned long addr)  in get_dump_page()
1003 	struct page *page;  in get_dump_page()  local
1006 			     FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma,  in get_dump_page()
1009 	flush_cache_page(vma, addr, page_to_pfn(page));  in get_dump_page()
1010 	return page;  in get_dump_page()
1054 			 int write, struct page **pages, int *nr)  in gup_pte_range()
1069 		struct page *page;  in gup_pte_range()  local
1080 		page = pte_page(pte);  in gup_pte_range()
1082 		if (!page_cache_get_speculative(page))  in gup_pte_range()
1086 			put_page(page);  in gup_pte_range()
1090 		pages[*nr] = page;  in gup_pte_range()
1113 			 int write, struct page **pages, int *nr)  in gup_pte_range()
1120 		unsigned long end, int write, struct page **pages, int *nr)  in gup_huge_pmd()
1122 	struct page *head, *page, *tail;  in gup_huge_pmd()  local
1130 	page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);  in gup_huge_pmd()
1131 	tail = page;  in gup_huge_pmd()
1133 		VM_BUG_ON_PAGE(compound_head(page) != head, page);  in gup_huge_pmd()
1134 		pages[*nr] = page;  in gup_huge_pmd()
1136 		page++;  in gup_huge_pmd()
1167 		unsigned long end, int write, struct page **pages, int *nr)  in gup_huge_pud()
1169 	struct page *head, *page, *tail;  in gup_huge_pud()  local
1177 	page = head + ((addr & ~PUD_MASK) >> PAGE_SHIFT);  in gup_huge_pud()
1178 	tail = page;  in gup_huge_pud()
1180 		VM_BUG_ON_PAGE(compound_head(page) != head, page);  in gup_huge_pud()
1181 		pages[*nr] = page;  in gup_huge_pud()
1183 		page++;  in gup_huge_pud()
1210 			struct page **pages, int *nr)  in gup_huge_pgd()
1213 	struct page *head, *page, *tail;  in gup_huge_pgd()  local
1220 	page = head + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);  in gup_huge_pgd()
1221 	tail = page;  in gup_huge_pgd()
1223 		VM_BUG_ON_PAGE(compound_head(page) != head, page);  in gup_huge_pgd()
1224 		pages[*nr] = page;  in gup_huge_pgd()
1226 		page++;  in gup_huge_pgd()
1252 		int write, struct page **pages, int *nr)  in gup_pmd_range()
1294 			 int write, struct page **pages, int *nr)  in gup_pud_range()
1326 			  struct page **pages)  in __get_user_pages_fast()
1396 			struct page **pages)  in get_user_pages_fast()