44 static void nfs_redirty_request(struct nfs_page *req);
49 static void nfs_clear_request_commit(struct nfs_page *req);
110 	struct nfs_page *req = NULL;  in nfs_page_find_head_request_locked()  local
113 		req = (struct nfs_page *)page_private(page);  in nfs_page_find_head_request_locked()
115 		req = nfs_page_search_commits_for_head_request_locked(nfsi,  in nfs_page_find_head_request_locked()
118 	if (req) {  in nfs_page_find_head_request_locked()
119 		WARN_ON_ONCE(req->wb_head != req);  in nfs_page_find_head_request_locked()
120 		kref_get(&req->wb_kref);  in nfs_page_find_head_request_locked()
123 	return req;  in nfs_page_find_head_request_locked()
134 	struct nfs_page *req = NULL;  in nfs_page_find_head_request()  local
137 	req = nfs_page_find_head_request_locked(NFS_I(inode), page);  in nfs_page_find_head_request()
139 	return req;  in nfs_page_find_head_request()
185 	struct nfs_page *req;  in nfs_page_group_search_locked()  local
190 	req = head;  in nfs_page_group_search_locked()
192 		if (page_offset >= req->wb_pgbase &&  in nfs_page_group_search_locked()
193 		    page_offset < (req->wb_pgbase + req->wb_bytes))  in nfs_page_group_search_locked()
194 			return req;  in nfs_page_group_search_locked()
196 		req = req->wb_this_page;  in nfs_page_group_search_locked()
197 	} while (req != head);  in nfs_page_group_search_locked()
209 static bool nfs_page_group_covers_page(struct nfs_page *req)  in nfs_page_group_covers_page()  argument
213 	unsigned int len = nfs_page_length(req->wb_page);  in nfs_page_group_covers_page()
215 	nfs_page_group_lock(req, false);  in nfs_page_group_covers_page()
218 		tmp = nfs_page_group_search_locked(req->wb_head, pos);  in nfs_page_group_covers_page()
226 	nfs_page_group_unlock(req);  in nfs_page_group_covers_page()
234 static void nfs_mark_uptodate(struct nfs_page *req)  in nfs_mark_uptodate()  argument
236 	if (PageUptodate(req->wb_page))  in nfs_mark_uptodate()
238 	if (!nfs_page_group_covers_page(req))  in nfs_mark_uptodate()
240 	SetPageUptodate(req->wb_page);  in nfs_mark_uptodate()
279 static void nfs_end_page_writeback(struct nfs_page *req)  in nfs_end_page_writeback()  argument
281 	struct inode *inode = page_file_mapping(req->wb_page)->host;  in nfs_end_page_writeback()
284 	if (!nfs_page_group_sync_on_bit(req, PG_WB_END))  in nfs_end_page_writeback()
287 	end_page_writeback(req->wb_page);  in nfs_end_page_writeback()
298 nfs_page_group_clear_bits(struct nfs_page *req)  in nfs_page_group_clear_bits()  argument
300 	clear_bit(PG_TEARDOWN, &req->wb_flags);  in nfs_page_group_clear_bits()
301 	clear_bit(PG_UNLOCKPAGE, &req->wb_flags);  in nfs_page_group_clear_bits()
302 	clear_bit(PG_UPTODATE, &req->wb_flags);  in nfs_page_group_clear_bits()
303 	clear_bit(PG_WB_END, &req->wb_flags);  in nfs_page_group_clear_bits()
304 	clear_bit(PG_REMOVE, &req->wb_flags);  in nfs_page_group_clear_bits()
325 			  struct nfs_page *req, bool nonblock)  in nfs_unroll_locks_and_wait()  argument
332 	for (tmp = head ; tmp != req; tmp = tmp->wb_this_page)  in nfs_unroll_locks_and_wait()
335 	WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));  in nfs_unroll_locks_and_wait()
338 	kref_get(&req->wb_kref);  in nfs_unroll_locks_and_wait()
347 		ret = nfs_wait_on_request(req);  in nfs_unroll_locks_and_wait()
350 	nfs_release_request(req);  in nfs_unroll_locks_and_wait()
555 	struct nfs_page *req;  in nfs_page_async_flush()  local
558 	req = nfs_lock_and_join_requests(page, nonblock);  in nfs_page_async_flush()
559 	if (!req)  in nfs_page_async_flush()
561 	ret = PTR_ERR(req);  in nfs_page_async_flush()
562 	if (IS_ERR(req))  in nfs_page_async_flush()
566 	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));  in nfs_page_async_flush()
569 	if (!nfs_pageio_add_request(pgio, req)) {  in nfs_page_async_flush()
570 		nfs_redirty_request(req);  in nfs_page_async_flush()
668 static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)  in nfs_inode_add_request()  argument
672 	WARN_ON_ONCE(req->wb_this_page != req);  in nfs_inode_add_request()
675 	nfs_lock_request(req);  in nfs_inode_add_request()
685 	if (likely(!PageSwapCache(req->wb_page))) {  in nfs_inode_add_request()
686 		set_bit(PG_MAPPED, &req->wb_flags);  in nfs_inode_add_request()
687 		SetPagePrivate(req->wb_page);  in nfs_inode_add_request()
688 		set_page_private(req->wb_page, (unsigned long)req);  in nfs_inode_add_request()
695 	WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));  in nfs_inode_add_request()
696 	kref_get(&req->wb_kref);  in nfs_inode_add_request()
703 static void nfs_inode_remove_request(struct nfs_page *req)  in nfs_inode_remove_request()  argument
705 	struct inode *inode = d_inode(req->wb_context->dentry);  in nfs_inode_remove_request()
709 	if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {  in nfs_inode_remove_request()
710 		head = req->wb_head;  in nfs_inode_remove_request()
728 	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags))  in nfs_inode_remove_request()
729 		nfs_release_request(req);  in nfs_inode_remove_request()
733 nfs_mark_request_dirty(struct nfs_page *req)  in nfs_mark_request_dirty()  argument
735 	__set_page_dirty_nobuffers(req->wb_page);  in nfs_mark_request_dirty()
783 nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,  in nfs_request_add_commit_list_locked()  argument
786 	set_bit(PG_CLEAN, &req->wb_flags);  in nfs_request_add_commit_list_locked()
787 	nfs_list_add_request(req, dst);  in nfs_request_add_commit_list_locked()
806 nfs_request_add_commit_list(struct nfs_page *req, struct list_head *dst,  in nfs_request_add_commit_list()  argument
810 	nfs_request_add_commit_list_locked(req, dst, cinfo);  in nfs_request_add_commit_list()
812 	nfs_mark_page_unstable(req->wb_page, cinfo);  in nfs_request_add_commit_list()
828 nfs_request_remove_commit_list(struct nfs_page *req,  in nfs_request_remove_commit_list()  argument
831 	if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))  in nfs_request_remove_commit_list()
833 	nfs_list_remove_request(req);  in nfs_request_remove_commit_list()
863 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,  in nfs_mark_request_commit()  argument
866 	if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))  in nfs_mark_request_commit()
868 	nfs_request_add_commit_list(req, &cinfo->mds->list, cinfo);  in nfs_mark_request_commit()
881 nfs_clear_request_commit(struct nfs_page *req)  in nfs_clear_request_commit()  argument
883 	if (test_bit(PG_CLEAN, &req->wb_flags)) {  in nfs_clear_request_commit()
884 		struct inode *inode = d_inode(req->wb_context->dentry);  in nfs_clear_request_commit()
888 		if (!pnfs_clear_request_commit(req, &cinfo)) {  in nfs_clear_request_commit()
889 			nfs_request_remove_commit_list(req, &cinfo);  in nfs_clear_request_commit()
891 		nfs_clear_page_commit(req->wb_page);  in nfs_clear_request_commit()
911 		struct nfs_page *req = nfs_list_entry(hdr->pages.next);  in nfs_write_completion()  local
913 		bytes += req->wb_bytes;  in nfs_write_completion()
914 		nfs_list_remove_request(req);  in nfs_write_completion()
917 			nfs_set_pageerror(req->wb_page);  in nfs_write_completion()
918 			nfs_context_set_write_error(req->wb_context, hdr->error);  in nfs_write_completion()
922 			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));  in nfs_write_completion()
923 			nfs_mark_request_commit(req, hdr->lseg, &cinfo,  in nfs_write_completion()
928 		nfs_inode_remove_request(req);  in nfs_write_completion()
930 		nfs_unlock_request(req);  in nfs_write_completion()
931 		nfs_end_page_writeback(req);  in nfs_write_completion()
932 		nfs_release_request(req);  in nfs_write_completion()
949 	struct nfs_page *req, *tmp;  in nfs_scan_commit_list()  local
952 	list_for_each_entry_safe(req, tmp, src, wb_list) {  in nfs_scan_commit_list()
953 		if (!nfs_lock_request(req))  in nfs_scan_commit_list()
955 		kref_get(&req->wb_kref);  in nfs_scan_commit_list()
957 			list_safe_reset_next(req, tmp, wb_list);  in nfs_scan_commit_list()
958 		nfs_request_remove_commit_list(req, cinfo);  in nfs_scan_commit_list()
959 		nfs_list_add_request(req, dst);  in nfs_scan_commit_list()
1006 	struct nfs_page *req;  in nfs_try_to_update_request()  local
1018 		req = nfs_page_find_head_request_locked(NFS_I(inode), page);  in nfs_try_to_update_request()
1019 		if (req == NULL)  in nfs_try_to_update_request()
1023 		WARN_ON_ONCE(req->wb_head != req);  in nfs_try_to_update_request()
1024 		WARN_ON_ONCE(req->wb_this_page != req);  in nfs_try_to_update_request()
1026 		rqend = req->wb_offset + req->wb_bytes;  in nfs_try_to_update_request()
1034 		    || end < req->wb_offset)  in nfs_try_to_update_request()
1037 		if (nfs_lock_request(req))  in nfs_try_to_update_request()
1042 		error = nfs_wait_on_request(req);  in nfs_try_to_update_request()
1043 		nfs_release_request(req);  in nfs_try_to_update_request()
1050 	if (offset < req->wb_offset) {  in nfs_try_to_update_request()
1051 		req->wb_offset = offset;  in nfs_try_to_update_request()
1052 		req->wb_pgbase = offset;  in nfs_try_to_update_request()
1055 		req->wb_bytes = end - req->wb_offset;  in nfs_try_to_update_request()
1057 		req->wb_bytes = rqend - req->wb_offset;  in nfs_try_to_update_request()
1059 	if (req)  in nfs_try_to_update_request()
1060 		nfs_clear_request_commit(req);  in nfs_try_to_update_request()
1062 	return req;  in nfs_try_to_update_request()
1065 	nfs_release_request(req);  in nfs_try_to_update_request()
1082 	struct nfs_page	*req;  in nfs_setup_write_request()  local
1084 	req = nfs_try_to_update_request(inode, page, offset, bytes);  in nfs_setup_write_request()
1085 	if (req != NULL)  in nfs_setup_write_request()
1087 	req = nfs_create_request(ctx, page, NULL, offset, bytes);  in nfs_setup_write_request()
1088 	if (IS_ERR(req))  in nfs_setup_write_request()
1090 	nfs_inode_add_request(inode, req);  in nfs_setup_write_request()
1092 	return req;  in nfs_setup_write_request()
1098 	struct nfs_page	*req;  in nfs_writepage_setup()  local
1100 	req = nfs_setup_write_request(ctx, page, offset, count);  in nfs_writepage_setup()
1101 	if (IS_ERR(req))  in nfs_writepage_setup()
1102 		return PTR_ERR(req);  in nfs_writepage_setup()
1105 	nfs_mark_uptodate(req);  in nfs_writepage_setup()
1106 	nfs_mark_request_dirty(req);  in nfs_writepage_setup()
1107 	nfs_unlock_and_release_request(req);  in nfs_writepage_setup()
1116 	struct nfs_page	*req;  in nfs_flush_incompatible()  local
1127 		req = nfs_page_find_head_request(page);  in nfs_flush_incompatible()
1128 		if (req == NULL)  in nfs_flush_incompatible()
1130 		l_ctx = req->wb_lock_context;  in nfs_flush_incompatible()
1131 		do_flush = req->wb_page != page || req->wb_context != ctx;  in nfs_flush_incompatible()
1133 		do_flush |= req->wb_this_page != req;  in nfs_flush_incompatible()
1140 		nfs_release_request(req);  in nfs_flush_incompatible()
1309 static void nfs_redirty_request(struct nfs_page *req)  in nfs_redirty_request()  argument
1311 	nfs_mark_request_dirty(req);  in nfs_redirty_request()
1312 	set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);  in nfs_redirty_request()
1313 	nfs_unlock_request(req);  in nfs_redirty_request()
1314 	nfs_end_page_writeback(req);  in nfs_redirty_request()
1315 	nfs_release_request(req);  in nfs_redirty_request()
1320 	struct nfs_page	*req;  in nfs_async_write_error()  local
1323 		req = nfs_list_entry(head->next);  in nfs_async_write_error()
1324 		nfs_list_remove_request(req);  in nfs_async_write_error()
1325 		nfs_redirty_request(req);  in nfs_async_write_error()
1605 	struct nfs_page *req;  in nfs_get_lwb()  local
1607 	list_for_each_entry(req, head, wb_list)  in nfs_get_lwb()
1608 		if (lwb < (req_offset(req) + req->wb_bytes))  in nfs_get_lwb()
1609 			lwb = req_offset(req) + req->wb_bytes;  in nfs_get_lwb()
1656 	struct nfs_page *req;  in nfs_retry_commit()  local
1659 		req = nfs_list_entry(page_list->next);  in nfs_retry_commit()
1660 		nfs_list_remove_request(req);  in nfs_retry_commit()
1661 		nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);  in nfs_retry_commit()
1663 			nfs_clear_page_commit(req->wb_page);  in nfs_retry_commit()
1664 		nfs_unlock_and_release_request(req);  in nfs_retry_commit()
1710 	struct nfs_page	*req;  in nfs_commit_release_pages()  local
1716 		req = nfs_list_entry(data->pages.next);  in nfs_commit_release_pages()
1717 		nfs_list_remove_request(req);  in nfs_commit_release_pages()
1718 		nfs_clear_page_commit(req->wb_page);  in nfs_commit_release_pages()
1721 			req->wb_context->dentry->d_sb->s_id,  in nfs_commit_release_pages()
1722 			(unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),  in nfs_commit_release_pages()
1723 			req->wb_bytes,  in nfs_commit_release_pages()
1724 			(long long)req_offset(req));  in nfs_commit_release_pages()
1726 			nfs_context_set_write_error(req->wb_context, status);  in nfs_commit_release_pages()
1727 			nfs_inode_remove_request(req);  in nfs_commit_release_pages()
1734 		if (!memcmp(&req->wb_verf, &data->verf.verifier, sizeof(req->wb_verf))) {  in nfs_commit_release_pages()
1736 			nfs_inode_remove_request(req);  in nfs_commit_release_pages()
1742 		nfs_mark_request_dirty(req);  in nfs_commit_release_pages()
1743 		set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);  in nfs_commit_release_pages()
1745 		nfs_unlock_and_release_request(req);  in nfs_commit_release_pages()
1888 	struct nfs_page *req;  in nfs_wb_page_cancel()  local
1895 	req = nfs_lock_and_join_requests(page, false);  in nfs_wb_page_cancel()
1897 	if (IS_ERR(req)) {  in nfs_wb_page_cancel()
1898 		ret = PTR_ERR(req);  in nfs_wb_page_cancel()
1899 	} else if (req) {  in nfs_wb_page_cancel()
1904 		nfs_inode_remove_request(req);  in nfs_wb_page_cancel()
1905 		nfs_unlock_and_release_request(req);  in nfs_wb_page_cancel()