112 static void page_cache_tree_delete(struct address_space *mapping,  in page_cache_tree_delete()  argument
123 	__radix_tree_lookup(&mapping->page_tree, page->index, &node, &slot);  in page_cache_tree_delete()
126 		mapping->nrshadows++;  in page_cache_tree_delete()
135 	mapping->nrpages--;  in page_cache_tree_delete()
139 		mapping->page_tree.gfp_mask &= __GFP_BITS_MASK;  in page_cache_tree_delete()
149 			radix_tree_tag_clear(&mapping->page_tree, index, tag);  in page_cache_tree_delete()
158 		if (__radix_tree_delete_node(&mapping->page_tree, node))  in page_cache_tree_delete()
170 		node->private_data = mapping;  in page_cache_tree_delete()
184 	struct address_space *mapping = page->mapping;  in __delete_from_page_cache()  local
195 		cleancache_invalidate_page(mapping, page);  in __delete_from_page_cache()
197 	page_cache_tree_delete(mapping, page, shadow);  in __delete_from_page_cache()
199 	page->mapping = NULL;  in __delete_from_page_cache()
218 		account_page_cleaned(page, mapping, memcg,  in __delete_from_page_cache()
219 				     inode_to_wb(mapping->host));  in __delete_from_page_cache()
232 	struct address_space *mapping = page->mapping;  in delete_from_page_cache()  local
240 	freepage = mapping->a_ops->freepage;  in delete_from_page_cache()
243 	spin_lock_irqsave(&mapping->tree_lock, flags);  in delete_from_page_cache()
245 	spin_unlock_irqrestore(&mapping->tree_lock, flags);  in delete_from_page_cache()
254 static int filemap_check_errors(struct address_space *mapping)  in filemap_check_errors()  argument
258 	if (test_bit(AS_ENOSPC, &mapping->flags) &&  in filemap_check_errors()
259 	    test_and_clear_bit(AS_ENOSPC, &mapping->flags))  in filemap_check_errors()
261 	if (test_bit(AS_EIO, &mapping->flags) &&  in filemap_check_errors()
262 	    test_and_clear_bit(AS_EIO, &mapping->flags))  in filemap_check_errors()
282 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,  in __filemap_fdatawrite_range()  argument
293 	if (!mapping_cap_writeback_dirty(mapping))  in __filemap_fdatawrite_range()
296 	wbc_attach_fdatawrite_inode(&wbc, mapping->host);  in __filemap_fdatawrite_range()
297 	ret = do_writepages(mapping, &wbc);  in __filemap_fdatawrite_range()
302 static inline int __filemap_fdatawrite(struct address_space *mapping,  in __filemap_fdatawrite()  argument
305 	return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);  in __filemap_fdatawrite()
308 int filemap_fdatawrite(struct address_space *mapping)  in filemap_fdatawrite()  argument
310 	return __filemap_fdatawrite(mapping, WB_SYNC_ALL);  in filemap_fdatawrite()
314 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,  in filemap_fdatawrite_range()  argument
317 	return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);  in filemap_fdatawrite_range()
328 int filemap_flush(struct address_space *mapping)  in filemap_flush()  argument
330 	return __filemap_fdatawrite(mapping, WB_SYNC_NONE);  in filemap_flush()
334 static int __filemap_fdatawait_range(struct address_space *mapping,  in __filemap_fdatawait_range()  argument
348 			(nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,  in __filemap_fdatawait_range()
385 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,  in filemap_fdatawait_range()  argument
390 	ret = __filemap_fdatawait_range(mapping, start_byte, end_byte);  in filemap_fdatawait_range()
391 	ret2 = filemap_check_errors(mapping);  in filemap_fdatawait_range()
411 void filemap_fdatawait_keep_errors(struct address_space *mapping)  in filemap_fdatawait_keep_errors()  argument
413 	loff_t i_size = i_size_read(mapping->host);  in filemap_fdatawait_keep_errors()
418 	__filemap_fdatawait_range(mapping, 0, i_size - 1);  in filemap_fdatawait_keep_errors()
433 int filemap_fdatawait(struct address_space *mapping)  in filemap_fdatawait()  argument
435 	loff_t i_size = i_size_read(mapping->host);  in filemap_fdatawait()
440 	return filemap_fdatawait_range(mapping, 0, i_size - 1);  in filemap_fdatawait()
444 int filemap_write_and_wait(struct address_space *mapping)  in filemap_write_and_wait()  argument
448 	if (mapping->nrpages) {  in filemap_write_and_wait()
449 		err = filemap_fdatawrite(mapping);  in filemap_write_and_wait()
457 			int err2 = filemap_fdatawait(mapping);  in filemap_write_and_wait()
462 		err = filemap_check_errors(mapping);  in filemap_write_and_wait()
479 int filemap_write_and_wait_range(struct address_space *mapping,  in filemap_write_and_wait_range()  argument
484 	if (mapping->nrpages) {  in filemap_write_and_wait_range()
485 		err = __filemap_fdatawrite_range(mapping, lstart, lend,  in filemap_write_and_wait_range()
489 			int err2 = filemap_fdatawait_range(mapping,  in filemap_write_and_wait_range()
495 		err = filemap_check_errors(mapping);  in filemap_write_and_wait_range()
522 	VM_BUG_ON_PAGE(new->mapping, new);  in replace_page_cache_page()
526 		struct address_space *mapping = old->mapping;  in replace_page_cache_page()  local
532 		freepage = mapping->a_ops->freepage;  in replace_page_cache_page()
535 		new->mapping = mapping;  in replace_page_cache_page()
539 		spin_lock_irqsave(&mapping->tree_lock, flags);  in replace_page_cache_page()
541 		error = radix_tree_insert(&mapping->page_tree, offset, new);  in replace_page_cache_page()
543 		mapping->nrpages++;  in replace_page_cache_page()
552 		spin_unlock_irqrestore(&mapping->tree_lock, flags);  in replace_page_cache_page()
565 static int page_cache_tree_insert(struct address_space *mapping,  in page_cache_tree_insert()  argument
572 	error = __radix_tree_create(&mapping->page_tree, page->index,  in page_cache_tree_insert()
579 		p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);  in page_cache_tree_insert()
584 		mapping->nrshadows--;  in page_cache_tree_insert()
589 	mapping->nrpages++;  in page_cache_tree_insert()
608 				      struct address_space *mapping,  in __add_to_page_cache_locked()  argument
634 	page->mapping = mapping;  in __add_to_page_cache_locked()
637 	spin_lock_irq(&mapping->tree_lock);  in __add_to_page_cache_locked()
638 	error = page_cache_tree_insert(mapping, page, shadowp);  in __add_to_page_cache_locked()
646 	spin_unlock_irq(&mapping->tree_lock);  in __add_to_page_cache_locked()
652 	page->mapping = NULL;  in __add_to_page_cache_locked()
654 	spin_unlock_irq(&mapping->tree_lock);  in __add_to_page_cache_locked()
671 int add_to_page_cache_locked(struct page *page, struct address_space *mapping,  in add_to_page_cache_locked()  argument
674 	return __add_to_page_cache_locked(page, mapping, offset,  in add_to_page_cache_locked()
679 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,  in add_to_page_cache_lru()  argument
686 	ret = __add_to_page_cache_locked(page, mapping, offset,  in add_to_page_cache_lru()
862 			if (page->mapping)  in page_endio()
863 				mapping_set_error(page->mapping, err);  in page_endio()
956 pgoff_t page_cache_next_hole(struct address_space *mapping,  in page_cache_next_hole()  argument
964 		page = radix_tree_lookup(&mapping->page_tree, index);  in page_cache_next_hole()
997 pgoff_t page_cache_prev_hole(struct address_space *mapping,  in page_cache_prev_hole()  argument
1005 		page = radix_tree_lookup(&mapping->page_tree, index);  in page_cache_prev_hole()
1030 struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)  in find_get_entry()  argument
1038 	pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);  in find_get_entry()
1089 struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)  in find_lock_entry()  argument
1094 	page = find_get_entry(mapping, offset);  in find_lock_entry()
1098 		if (unlikely(page->mapping != mapping)) {  in find_lock_entry()
1132 struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,  in pagecache_get_page()  argument
1138 	page = find_get_entry(mapping, offset);  in pagecache_get_page()
1155 		if (unlikely(page->mapping != mapping)) {  in pagecache_get_page()
1169 		if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))  in pagecache_get_page()
1185 		err = add_to_page_cache_lru(page, mapping, offset,  in pagecache_get_page()
1222 unsigned find_get_entries(struct address_space *mapping,  in find_get_entries()  argument
1235 	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {  in find_get_entries()
1285 unsigned find_get_pages(struct address_space *mapping, pgoff_t start,  in find_get_pages()  argument
1297 	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {  in find_get_pages()
1352 unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,  in find_get_pages_contig()  argument
1364 	radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {  in find_get_pages_contig()
1403 		if (page->mapping == NULL || page->index != iter.index) {  in find_get_pages_contig()
1428 unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,  in find_get_pages_tag()  argument
1440 	radix_tree_for_each_tagged(slot, &mapping->page_tree,  in find_get_pages_tag()
1531 	struct address_space *mapping = filp->f_mapping;  in do_generic_file_read()  local
1532 	struct inode *inode = mapping->host;  in do_generic_file_read()
1555 		page = find_get_page(mapping, index);  in do_generic_file_read()
1557 			page_cache_sync_readahead(mapping,  in do_generic_file_read()
1560 			page = find_get_page(mapping, index);  in do_generic_file_read()
1565 			page_cache_async_readahead(mapping,  in do_generic_file_read()
1571 					!mapping->a_ops->is_partially_uptodate)  in do_generic_file_read()
1576 			if (!page->mapping)  in do_generic_file_read()
1578 			if (!mapping->a_ops->is_partially_uptodate(page,  in do_generic_file_read()
1615 		if (mapping_writably_mapped(mapping))  in do_generic_file_read()
1655 		if (!page->mapping) {  in do_generic_file_read()
1675 		error = mapping->a_ops->readpage(filp, page);  in do_generic_file_read()
1691 				if (page->mapping == NULL) {  in do_generic_file_read()
1719 		page = page_cache_alloc_cold(mapping);  in do_generic_file_read()
1724 		error = add_to_page_cache_lru(page, mapping, index,  in do_generic_file_read()
1725 				mapping_gfp_constraint(mapping, GFP_KERNEL));  in do_generic_file_read()
1764 		struct address_space *mapping = file->f_mapping;  in generic_file_read_iter()  local
1765 		struct inode *inode = mapping->host;  in generic_file_read_iter()
1772 		retval = filemap_write_and_wait_range(mapping, pos,  in generic_file_read_iter()
1776 			retval = mapping->a_ops->direct_IO(iocb, &data, pos);  in generic_file_read_iter()
1817 	struct address_space *mapping = file->f_mapping;  in page_cache_read()  local
1822 		page = page_cache_alloc_cold(mapping);  in page_cache_read()
1826 		ret = add_to_page_cache_lru(page, mapping, offset,  in page_cache_read()
1827 				mapping_gfp_constraint(mapping, GFP_KERNEL));  in page_cache_read()
1829 			ret = mapping->a_ops->readpage(file, page);  in page_cache_read()
1851 	struct address_space *mapping = file->f_mapping;  in do_sync_mmap_readahead()  local
1860 		page_cache_sync_readahead(mapping, ra, file, offset,  in do_sync_mmap_readahead()
1882 	ra_submit(ra, mapping, file);  in do_sync_mmap_readahead()
1895 	struct address_space *mapping = file->f_mapping;  in do_async_mmap_readahead()  local
1903 		page_cache_async_readahead(mapping, ra, file,  in do_async_mmap_readahead()
1935 	struct address_space *mapping = file->f_mapping;  in filemap_fault()  local
1937 	struct inode *inode = mapping->host;  in filemap_fault()
1950 	page = find_get_page(mapping, offset);  in filemap_fault()
1964 		page = find_get_page(mapping, offset);  in filemap_fault()
1975 	if (unlikely(page->mapping != mapping)) {  in filemap_fault()
2035 	error = mapping->a_ops->readpage(file, page);  in filemap_fault()
2057 	struct address_space *mapping = file->f_mapping;  in filemap_map_pages()  local
2065 	radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, vmf->pgoff) {  in filemap_map_pages()
2095 		if (page->mapping != mapping || !PageUptodate(page))  in filemap_map_pages()
2098 		size = round_up(i_size_read(mapping->host), PAGE_CACHE_SIZE);  in filemap_map_pages()
2133 	if (page->mapping != inode->i_mapping) {  in filemap_page_mkwrite()
2161 	struct address_space *mapping = file->f_mapping;  in generic_file_mmap()  local
2163 	if (!mapping->a_ops->readpage)  in generic_file_mmap()
2205 static struct page *__read_cache_page(struct address_space *mapping,  in __read_cache_page()  argument
2214 	page = find_get_page(mapping, index);  in __read_cache_page()
2219 		err = add_to_page_cache_lru(page, mapping, index, gfp);  in __read_cache_page()
2238 static struct page *do_read_cache_page(struct address_space *mapping,  in do_read_cache_page()  argument
2249 	page = __read_cache_page(mapping, index, filler, data, gfp);  in do_read_cache_page()
2256 	if (!page->mapping) {  in do_read_cache_page()
2291 struct page *read_cache_page(struct address_space *mapping,  in read_cache_page()  argument
2296 	return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));  in read_cache_page()
2311 struct page *read_cache_page_gfp(struct address_space *mapping,  in read_cache_page_gfp()  argument
2315 	filler_t *filler = (filler_t *)mapping->a_ops->readpage;  in read_cache_page_gfp()
2317 	return do_read_cache_page(mapping, index, filler, NULL, gfp);  in read_cache_page_gfp()
2377 int pagecache_write_begin(struct file *file, struct address_space *mapping,  in pagecache_write_begin()  argument
2381 	const struct address_space_operations *aops = mapping->a_ops;  in pagecache_write_begin()
2383 	return aops->write_begin(file, mapping, pos, len, flags,  in pagecache_write_begin()
2388 int pagecache_write_end(struct file *file, struct address_space *mapping,  in pagecache_write_end()  argument
2392 	const struct address_space_operations *aops = mapping->a_ops;  in pagecache_write_end()
2394 	return aops->write_end(file, mapping, pos, len, copied, page, fsdata);  in pagecache_write_end()
2402 	struct address_space *mapping = file->f_mapping;  in generic_file_direct_write()  local
2403 	struct inode	*inode = mapping->host;  in generic_file_direct_write()
2412 	written = filemap_write_and_wait_range(mapping, pos, pos + write_len - 1);  in generic_file_direct_write()
2422 	if (mapping->nrpages) {  in generic_file_direct_write()
2423 		written = invalidate_inode_pages2_range(mapping,  in generic_file_direct_write()
2437 	written = mapping->a_ops->direct_IO(iocb, &data, pos);  in generic_file_direct_write()
2447 	if (mapping->nrpages) {  in generic_file_direct_write()
2448 		invalidate_inode_pages2_range(mapping,  in generic_file_direct_write()
2470 struct page *grab_cache_page_write_begin(struct address_space *mapping,  in grab_cache_page_write_begin()  argument
2479 	page = pagecache_get_page(mapping, index, fgp_flags,  in grab_cache_page_write_begin()
2480 			mapping_gfp_mask(mapping));  in grab_cache_page_write_begin()
2491 	struct address_space *mapping = file->f_mapping;  in generic_perform_write()  local
2492 	const struct address_space_operations *a_ops = mapping->a_ops;  in generic_perform_write()
2535 		status = a_ops->write_begin(file, mapping, pos, bytes, flags,  in generic_perform_write()
2540 		if (mapping_writably_mapped(mapping))  in generic_perform_write()
2546 		status = a_ops->write_end(file, mapping, pos, bytes, copied,  in generic_perform_write()
2571 		balance_dirty_pages_ratelimited(mapping);  in generic_perform_write()
2598 	struct address_space * mapping = file->f_mapping;  in __generic_file_write_iter()  local
2599 	struct inode 	*inode = mapping->host;  in __generic_file_write_iter()
2646 		err = filemap_write_and_wait_range(mapping, pos, endbyte);  in __generic_file_write_iter()
2650 			invalidate_mapping_pages(mapping,  in __generic_file_write_iter()
2721 	struct address_space * const mapping = page->mapping;  in try_to_release_page()  local
2727 	if (mapping && mapping->a_ops->releasepage)  in try_to_release_page()
2728 		return mapping->a_ops->releasepage(page, gfp_mask);  in try_to_release_page()