119 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
120 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
123 	struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type);
126 	struct page **pagep, enum sgp_type sgp, int *fault_type)  in shmem_getpage()
292 static int shmem_add_to_page_cache(struct page *page,  in shmem_add_to_page_cache()  argument
298 	VM_BUG_ON_PAGE(!PageLocked(page), page);  in shmem_add_to_page_cache()
299 	VM_BUG_ON_PAGE(!PageSwapBacked(page), page);  in shmem_add_to_page_cache()
301 	page_cache_get(page);  in shmem_add_to_page_cache()
302 	page->mapping = mapping;  in shmem_add_to_page_cache()
303 	page->index = index;  in shmem_add_to_page_cache()
307 		error = radix_tree_insert(&mapping->page_tree, index, page);  in shmem_add_to_page_cache()
310 								 page);  in shmem_add_to_page_cache()
313 		__inc_zone_page_state(page, NR_FILE_PAGES);  in shmem_add_to_page_cache()
314 		__inc_zone_page_state(page, NR_SHMEM);  in shmem_add_to_page_cache()
317 		page->mapping = NULL;  in shmem_add_to_page_cache()
319 		page_cache_release(page);  in shmem_add_to_page_cache()
327 static void shmem_delete_from_page_cache(struct page *page, void *radswap)  in shmem_delete_from_page_cache()  argument
329 	struct address_space *mapping = page->mapping;  in shmem_delete_from_page_cache()
333 	error = shmem_radix_tree_replace(mapping, page->index, page, radswap);  in shmem_delete_from_page_cache()
334 	page->mapping = NULL;  in shmem_delete_from_page_cache()
336 	__dec_zone_page_state(page, NR_FILE_PAGES);  in shmem_delete_from_page_cache()
337 	__dec_zone_page_state(page, NR_SHMEM);  in shmem_delete_from_page_cache()
339 	page_cache_release(page);  in shmem_delete_from_page_cache()
421 			struct page *page = pvec.pages[i];  in shmem_undo_range()  local
427 			if (radix_tree_exceptional_entry(page)) {  in shmem_undo_range()
431 								index, page);  in shmem_undo_range()
435 			if (!trylock_page(page))  in shmem_undo_range()
437 			if (!unfalloc || !PageUptodate(page)) {  in shmem_undo_range()
438 				if (page->mapping == mapping) {  in shmem_undo_range()
439 					VM_BUG_ON_PAGE(PageWriteback(page), page);  in shmem_undo_range()
440 					truncate_inode_page(mapping, page);  in shmem_undo_range()
443 			unlock_page(page);  in shmem_undo_range()
452 		struct page *page = NULL;  in shmem_undo_range()  local
453 		shmem_getpage(inode, start - 1, &page, SGP_READ, NULL);  in shmem_undo_range()
454 		if (page) {  in shmem_undo_range()
460 			zero_user_segment(page, partial_start, top);  in shmem_undo_range()
461 			set_page_dirty(page);  in shmem_undo_range()
462 			unlock_page(page);  in shmem_undo_range()
463 			page_cache_release(page);  in shmem_undo_range()
467 		struct page *page = NULL;  in shmem_undo_range()  local
468 		shmem_getpage(inode, end, &page, SGP_READ, NULL);  in shmem_undo_range()
469 		if (page) {  in shmem_undo_range()
470 			zero_user_segment(page, 0, partial_end);  in shmem_undo_range()
471 			set_page_dirty(page);  in shmem_undo_range()
472 			unlock_page(page);  in shmem_undo_range()
473 			page_cache_release(page);  in shmem_undo_range()
495 			struct page *page = pvec.pages[i];  in shmem_undo_range()  local
501 			if (radix_tree_exceptional_entry(page)) {  in shmem_undo_range()
504 				if (shmem_free_swap(mapping, index, page)) {  in shmem_undo_range()
513 			lock_page(page);  in shmem_undo_range()
514 			if (!unfalloc || !PageUptodate(page)) {  in shmem_undo_range()
515 				if (page->mapping == mapping) {  in shmem_undo_range()
516 					VM_BUG_ON_PAGE(PageWriteback(page), page);  in shmem_undo_range()
517 					truncate_inode_page(mapping, page);  in shmem_undo_range()
520 					unlock_page(page);  in shmem_undo_range()
525 			unlock_page(page);  in shmem_undo_range()
613 			     swp_entry_t swap, struct page **pagep)  in shmem_unuse_inode()
690 int shmem_unuse(swp_entry_t swap, struct page *page)  in shmem_unuse()  argument
701 	if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))  in shmem_unuse()
709 	error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg);  in shmem_unuse()
719 			error = shmem_unuse_inode(info, swap, &page);  in shmem_unuse()
732 		mem_cgroup_cancel_charge(page, memcg);  in shmem_unuse()
734 		mem_cgroup_commit_charge(page, memcg, true);  in shmem_unuse()
736 	unlock_page(page);  in shmem_unuse()
737 	page_cache_release(page);  in shmem_unuse()
744 static int shmem_writepage(struct page *page, struct writeback_control *wbc)  in shmem_writepage()  argument
752 	BUG_ON(!PageLocked(page));  in shmem_writepage()
753 	mapping = page->mapping;  in shmem_writepage()
754 	index = page->index;  in shmem_writepage()
785 	if (!PageUptodate(page)) {  in shmem_writepage()
801 		clear_highpage(page);  in shmem_writepage()
802 		flush_dcache_page(page);  in shmem_writepage()
803 		SetPageUptodate(page);  in shmem_writepage()
822 	if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {  in shmem_writepage()
824 		shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));  in shmem_writepage()
832 		BUG_ON(page_mapped(page));  in shmem_writepage()
833 		swap_writepage(page, wbc);  in shmem_writepage()
840 	set_page_dirty(page);  in shmem_writepage()
843 	unlock_page(page);  in shmem_writepage()
874 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,  in shmem_swapin()
878 	struct page *page;  in shmem_swapin()  local
887 	page = swapin_readahead(swap, gfp, &pvma, 0);  in shmem_swapin()
892 	return page;  in shmem_swapin()
895 static struct page *shmem_alloc_page(gfp_t gfp,  in shmem_alloc_page()
899 	struct page *page;  in shmem_alloc_page()  local
908 	page = alloc_page_vma(gfp, &pvma, 0);  in shmem_alloc_page()
913 	return page;  in shmem_alloc_page()
922 static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,  in shmem_swapin()
928 static inline struct page *shmem_alloc_page(gfp_t gfp,  in shmem_alloc_page()
954 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)  in shmem_should_replace_page()  argument
956 	return page_zonenum(page) > gfp_zone(gfp);  in shmem_should_replace_page()
959 static int shmem_replace_page(struct page **pagep, gfp_t gfp,  in shmem_replace_page()
962 	struct page *oldpage, *newpage;  in shmem_replace_page()
1033 	struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type)  in shmem_getpage_gfp()
1039 	struct page *page;  in shmem_getpage_gfp()  local
1049 	page = find_lock_entry(mapping, index);  in shmem_getpage_gfp()
1050 	if (radix_tree_exceptional_entry(page)) {  in shmem_getpage_gfp()
1051 		swap = radix_to_swp_entry(page);  in shmem_getpage_gfp()
1052 		page = NULL;  in shmem_getpage_gfp()
1061 	if (page && sgp == SGP_WRITE)  in shmem_getpage_gfp()
1062 		mark_page_accessed(page);  in shmem_getpage_gfp()
1065 	if (page && !PageUptodate(page)) {  in shmem_getpage_gfp()
1068 		unlock_page(page);  in shmem_getpage_gfp()
1069 		page_cache_release(page);  in shmem_getpage_gfp()
1070 		page = NULL;  in shmem_getpage_gfp()
1072 	if (page || (sgp == SGP_READ && !swap.val)) {  in shmem_getpage_gfp()
1073 		*pagep = page;  in shmem_getpage_gfp()
1086 		page = lookup_swap_cache(swap);  in shmem_getpage_gfp()
1087 		if (!page) {  in shmem_getpage_gfp()
1091 			page = shmem_swapin(swap, gfp, info, index);  in shmem_getpage_gfp()
1092 			if (!page) {  in shmem_getpage_gfp()
1099 		lock_page(page);  in shmem_getpage_gfp()
1100 		if (!PageSwapCache(page) || page_private(page) != swap.val ||  in shmem_getpage_gfp()
1105 		if (!PageUptodate(page)) {  in shmem_getpage_gfp()
1109 		wait_on_page_writeback(page);  in shmem_getpage_gfp()
1111 		if (shmem_should_replace_page(page, gfp)) {  in shmem_getpage_gfp()
1112 			error = shmem_replace_page(&page, gfp, info, index);  in shmem_getpage_gfp()
1117 		error = mem_cgroup_try_charge(page, current->mm, gfp, &memcg);  in shmem_getpage_gfp()
1119 			error = shmem_add_to_page_cache(page, mapping, index,  in shmem_getpage_gfp()
1134 				mem_cgroup_cancel_charge(page, memcg);  in shmem_getpage_gfp()
1135 				delete_from_swap_cache(page);  in shmem_getpage_gfp()
1141 		mem_cgroup_commit_charge(page, memcg, true);  in shmem_getpage_gfp()
1149 			mark_page_accessed(page);  in shmem_getpage_gfp()
1151 		delete_from_swap_cache(page);  in shmem_getpage_gfp()
1152 		set_page_dirty(page);  in shmem_getpage_gfp()
1169 		page = shmem_alloc_page(gfp, info, index);  in shmem_getpage_gfp()
1170 		if (!page) {  in shmem_getpage_gfp()
1175 		__SetPageSwapBacked(page);  in shmem_getpage_gfp()
1176 		__set_page_locked(page);  in shmem_getpage_gfp()
1178 			__SetPageReferenced(page);  in shmem_getpage_gfp()
1180 		error = mem_cgroup_try_charge(page, current->mm, gfp, &memcg);  in shmem_getpage_gfp()
1185 			error = shmem_add_to_page_cache(page, mapping, index,  in shmem_getpage_gfp()
1190 			mem_cgroup_cancel_charge(page, memcg);  in shmem_getpage_gfp()
1193 		mem_cgroup_commit_charge(page, memcg, false);  in shmem_getpage_gfp()
1194 		lru_cache_add_anon(page);  in shmem_getpage_gfp()
1215 			clear_highpage(page);  in shmem_getpage_gfp()
1216 			flush_dcache_page(page);  in shmem_getpage_gfp()
1217 			SetPageUptodate(page);  in shmem_getpage_gfp()
1220 			set_page_dirty(page);  in shmem_getpage_gfp()
1232 	*pagep = page;  in shmem_getpage_gfp()
1240 	ClearPageDirty(page);  in shmem_getpage_gfp()
1241 	delete_from_page_cache(page);  in shmem_getpage_gfp()
1257 	if (page) {  in shmem_getpage_gfp()
1258 		unlock_page(page);  in shmem_getpage_gfp()
1259 		page_cache_release(page);  in shmem_getpage_gfp()
1337 	error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);  in shmem_fault()
1477 			struct page **pagep, void **fsdata)  in shmem_write_begin()
1497 			struct page *page, void *fsdata)  in shmem_write_end()  argument
1504 	if (!PageUptodate(page)) {  in shmem_write_end()
1507 			zero_user_segments(page, 0, from,  in shmem_write_end()
1510 		SetPageUptodate(page);  in shmem_write_end()
1512 	set_page_dirty(page);  in shmem_write_end()
1513 	unlock_page(page);  in shmem_write_end()
1514 	page_cache_release(page);  in shmem_write_end()
1543 		struct page *page = NULL;  in shmem_file_read_iter()  local
1557 		error = shmem_getpage(inode, index, &page, sgp, NULL);  in shmem_file_read_iter()
1563 		if (page)  in shmem_file_read_iter()
1564 			unlock_page(page);  in shmem_file_read_iter()
1576 				if (page)  in shmem_file_read_iter()
1577 					page_cache_release(page);  in shmem_file_read_iter()
1583 		if (page) {  in shmem_file_read_iter()
1590 				flush_dcache_page(page);  in shmem_file_read_iter()
1595 				mark_page_accessed(page);  in shmem_file_read_iter()
1597 			page = ZERO_PAGE(0);  in shmem_file_read_iter()
1598 			page_cache_get(page);  in shmem_file_read_iter()
1605 		ret = copy_page_to_iter(page, offset, nr, to);  in shmem_file_read_iter()
1611 		page_cache_release(page);  in shmem_file_read_iter()
1633 	struct page *pages[PIPE_DEF_BUFFERS];  in shmem_file_splice_read()
1635 	struct page *page;  in shmem_file_splice_read()  local
1670 		error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL);  in shmem_file_splice_read()
1673 		unlock_page(page);  in shmem_file_splice_read()
1674 		spd.pages[spd.nr_pages++] = page;  in shmem_file_splice_read()
1689 		page = spd.pages[page_nr];  in shmem_file_splice_read()
1691 		if (!PageUptodate(page) || page->mapping != mapping) {  in shmem_file_splice_read()
1692 			error = shmem_getpage(inode, index, &page,  in shmem_file_splice_read()
1696 			unlock_page(page);  in shmem_file_splice_read()
1698 			spd.pages[page_nr] = page;  in shmem_file_splice_read()
1746 	struct page *page;  in shmem_seek_hole_data()  local
1770 			page = pvec.pages[i];  in shmem_seek_hole_data()
1771 			if (page && !radix_tree_exceptional_entry(page)) {  in shmem_seek_hole_data()
1772 				if (!PageUptodate(page))  in shmem_seek_hole_data()
1773 					page = NULL;  in shmem_seek_hole_data()
1776 			    (page && whence == SEEK_DATA) ||  in shmem_seek_hole_data()
1777 			    (!page && whence == SEEK_HOLE)) {  in shmem_seek_hole_data()
1840 	struct page *page;  in shmem_tag_pins()  local
1848 		page = radix_tree_deref_slot(slot);  in shmem_tag_pins()
1849 		if (!page || radix_tree_exception(page)) {  in shmem_tag_pins()
1850 			if (radix_tree_deref_retry(page))  in shmem_tag_pins()
1852 		} else if (page_count(page) - page_mapcount(page) > 1) {  in shmem_tag_pins()
1882 	struct page *page;  in shmem_wait_for_pins()  local
1903 			page = radix_tree_deref_slot(slot);  in shmem_wait_for_pins()
1904 			if (radix_tree_exception(page)) {  in shmem_wait_for_pins()
1905 				if (radix_tree_deref_retry(page))  in shmem_wait_for_pins()
1908 				page = NULL;  in shmem_wait_for_pins()
1911 			if (page &&  in shmem_wait_for_pins()
1912 			    page_count(page) - page_mapcount(page) != 1) {  in shmem_wait_for_pins()
2125 		struct page *page;  in shmem_fallocate()  local
2136 			error = shmem_getpage(inode, index, &page, SGP_FALLOC,  in shmem_fallocate()
2151 		if (!PageUptodate(page))  in shmem_fallocate()
2161 		set_page_dirty(page);  in shmem_fallocate()
2162 		unlock_page(page);  in shmem_fallocate()
2163 		page_cache_release(page);  in shmem_fallocate()
2423 	struct page *page;  in shmem_symlink()  local
2455 		error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);  in shmem_symlink()
2462 		kaddr = kmap_atomic(page);  in shmem_symlink()
2465 		SetPageUptodate(page);  in shmem_symlink()
2466 		set_page_dirty(page);  in shmem_symlink()
2467 		unlock_page(page);  in shmem_symlink()
2468 		page_cache_release(page);  in shmem_symlink()
2485 	struct page *page = NULL;  in shmem_follow_link()  local
2486 	int error = shmem_getpage(d_inode(dentry), 0, &page, SGP_READ, NULL);  in shmem_follow_link()
2487 	nd_set_link(nd, error ? ERR_PTR(error) : kmap(page));  in shmem_follow_link()
2488 	if (page)  in shmem_follow_link()
2489 		unlock_page(page);  in shmem_follow_link()
2490 	return page;  in shmem_follow_link()
2496 		struct page *page = cookie;  in shmem_put_link()  local
2497 		kunmap(page);  in shmem_put_link()
2498 		mark_page_accessed(page);  in shmem_put_link()
2499 		page_cache_release(page);  in shmem_put_link()
3278 int shmem_unuse(swp_entry_t swap, struct page *page)  in shmem_unuse()  argument
3436 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,  in shmem_read_mapping_page_gfp()
3441 	struct page *page;  in shmem_read_mapping_page_gfp()  local
3445 	error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL);  in shmem_read_mapping_page_gfp()
3447 		page = ERR_PTR(error);  in shmem_read_mapping_page_gfp()
3449 		unlock_page(page);  in shmem_read_mapping_page_gfp()
3450 	return page;  in shmem_read_mapping_page_gfp()