44 static void nfs_redirty_request(struct nfs_page *req);
49 static void nfs_clear_request_commit(struct nfs_page *req);
110 	struct nfs_page *req = NULL;  in nfs_page_find_head_request_locked()  local
113 		req = (struct nfs_page *)page_private(page);  in nfs_page_find_head_request_locked()
115 		req = nfs_page_search_commits_for_head_request_locked(nfsi,  in nfs_page_find_head_request_locked()
118 	if (req) {  in nfs_page_find_head_request_locked()
119 		WARN_ON_ONCE(req->wb_head != req);  in nfs_page_find_head_request_locked()
120 		kref_get(&req->wb_kref);  in nfs_page_find_head_request_locked()
123 	return req;  in nfs_page_find_head_request_locked()
134 	struct nfs_page *req = NULL;  in nfs_page_find_head_request()  local
137 	req = nfs_page_find_head_request_locked(NFS_I(inode), page);  in nfs_page_find_head_request()
139 	return req;  in nfs_page_find_head_request()
185 	struct nfs_page *req;  in nfs_page_group_search_locked()  local
190 	req = head;  in nfs_page_group_search_locked()
192 		if (page_offset >= req->wb_pgbase &&  in nfs_page_group_search_locked()
193 		    page_offset < (req->wb_pgbase + req->wb_bytes))  in nfs_page_group_search_locked()
194 			return req;  in nfs_page_group_search_locked()
196 		req = req->wb_this_page;  in nfs_page_group_search_locked()
197 	} while (req != head);  in nfs_page_group_search_locked()
209 static bool nfs_page_group_covers_page(struct nfs_page *req)  in nfs_page_group_covers_page()  argument
213 	unsigned int len = nfs_page_length(req->wb_page);  in nfs_page_group_covers_page()
215 	nfs_page_group_lock(req, false);  in nfs_page_group_covers_page()
218 		tmp = nfs_page_group_search_locked(req->wb_head, pos);  in nfs_page_group_covers_page()
226 	nfs_page_group_unlock(req);  in nfs_page_group_covers_page()
234 static void nfs_mark_uptodate(struct nfs_page *req)  in nfs_mark_uptodate()  argument
236 	if (PageUptodate(req->wb_page))  in nfs_mark_uptodate()
238 	if (!nfs_page_group_covers_page(req))  in nfs_mark_uptodate()
240 	SetPageUptodate(req->wb_page);  in nfs_mark_uptodate()
279 static void nfs_end_page_writeback(struct nfs_page *req)  in nfs_end_page_writeback()  argument
281 	struct inode *inode = page_file_mapping(req->wb_page)->host;  in nfs_end_page_writeback()
284 	if (!nfs_page_group_sync_on_bit(req, PG_WB_END))  in nfs_end_page_writeback()
287 	end_page_writeback(req->wb_page);  in nfs_end_page_writeback()
298 nfs_page_group_clear_bits(struct nfs_page *req)  in nfs_page_group_clear_bits()  argument
300 	clear_bit(PG_TEARDOWN, &req->wb_flags);  in nfs_page_group_clear_bits()
301 	clear_bit(PG_UNLOCKPAGE, &req->wb_flags);  in nfs_page_group_clear_bits()
302 	clear_bit(PG_UPTODATE, &req->wb_flags);  in nfs_page_group_clear_bits()
303 	clear_bit(PG_WB_END, &req->wb_flags);  in nfs_page_group_clear_bits()
304 	clear_bit(PG_REMOVE, &req->wb_flags);  in nfs_page_group_clear_bits()
325 			  struct nfs_page *req, bool nonblock)  in nfs_unroll_locks_and_wait()  argument
332 	for (tmp = head ; tmp != req; tmp = tmp->wb_this_page)  in nfs_unroll_locks_and_wait()
335 	WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));  in nfs_unroll_locks_and_wait()
338 	kref_get(&req->wb_kref);  in nfs_unroll_locks_and_wait()
347 		ret = nfs_wait_on_request(req);  in nfs_unroll_locks_and_wait()
350 	nfs_release_request(req);  in nfs_unroll_locks_and_wait()
555 	struct nfs_page *req;  in nfs_page_async_flush()  local
558 	req = nfs_lock_and_join_requests(page, nonblock);  in nfs_page_async_flush()
559 	if (!req)  in nfs_page_async_flush()
561 	ret = PTR_ERR(req);  in nfs_page_async_flush()
562 	if (IS_ERR(req))  in nfs_page_async_flush()
566 	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));  in nfs_page_async_flush()
569 	if (!nfs_pageio_add_request(pgio, req)) {  in nfs_page_async_flush()
570 		nfs_redirty_request(req);  in nfs_page_async_flush()
668 static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)  in nfs_inode_add_request()  argument
672 	WARN_ON_ONCE(req->wb_this_page != req);  in nfs_inode_add_request()
675 	nfs_lock_request(req);  in nfs_inode_add_request()
685 	if (likely(!PageSwapCache(req->wb_page))) {  in nfs_inode_add_request()
686 		set_bit(PG_MAPPED, &req->wb_flags);  in nfs_inode_add_request()
687 		SetPagePrivate(req->wb_page);  in nfs_inode_add_request()
688 		set_page_private(req->wb_page, (unsigned long)req);  in nfs_inode_add_request()
695 	WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));  in nfs_inode_add_request()
696 	kref_get(&req->wb_kref);  in nfs_inode_add_request()
703 static void nfs_inode_remove_request(struct nfs_page *req)  in nfs_inode_remove_request()  argument
705 	struct inode *inode = d_inode(req->wb_context->dentry);  in nfs_inode_remove_request()
709 	if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {  in nfs_inode_remove_request()
710 		head = req->wb_head;  in nfs_inode_remove_request()
728 	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags))  in nfs_inode_remove_request()
729 		nfs_release_request(req);  in nfs_inode_remove_request()
733 nfs_mark_request_dirty(struct nfs_page *req)  in nfs_mark_request_dirty()  argument
735 	__set_page_dirty_nobuffers(req->wb_page);  in nfs_mark_request_dirty()
784 nfs_request_add_commit_list(struct nfs_page *req, struct list_head *dst,  in nfs_request_add_commit_list()  argument
787 	set_bit(PG_CLEAN, &(req)->wb_flags);  in nfs_request_add_commit_list()
789 	nfs_list_add_request(req, dst);  in nfs_request_add_commit_list()
793 		nfs_mark_page_unstable(req->wb_page);  in nfs_request_add_commit_list()
809 nfs_request_remove_commit_list(struct nfs_page *req,  in nfs_request_remove_commit_list()  argument
812 	if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))  in nfs_request_remove_commit_list()
814 	nfs_list_remove_request(req);  in nfs_request_remove_commit_list()
844 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,  in nfs_mark_request_commit()  argument
847 	if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))  in nfs_mark_request_commit()
849 	nfs_request_add_commit_list(req, &cinfo->mds->list, cinfo);  in nfs_mark_request_commit()
861 nfs_clear_request_commit(struct nfs_page *req)  in nfs_clear_request_commit()  argument
863 	if (test_bit(PG_CLEAN, &req->wb_flags)) {  in nfs_clear_request_commit()
864 		struct inode *inode = d_inode(req->wb_context->dentry);  in nfs_clear_request_commit()
868 		if (!pnfs_clear_request_commit(req, &cinfo)) {  in nfs_clear_request_commit()
869 			nfs_request_remove_commit_list(req, &cinfo);  in nfs_clear_request_commit()
871 		nfs_clear_page_commit(req->wb_page);  in nfs_clear_request_commit()
891 		struct nfs_page *req = nfs_list_entry(hdr->pages.next);  in nfs_write_completion()  local
893 		bytes += req->wb_bytes;  in nfs_write_completion()
894 		nfs_list_remove_request(req);  in nfs_write_completion()
897 			nfs_set_pageerror(req->wb_page);  in nfs_write_completion()
898 			nfs_context_set_write_error(req->wb_context, hdr->error);  in nfs_write_completion()
902 			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));  in nfs_write_completion()
903 			nfs_mark_request_commit(req, hdr->lseg, &cinfo,  in nfs_write_completion()
908 		nfs_inode_remove_request(req);  in nfs_write_completion()
910 		nfs_unlock_request(req);  in nfs_write_completion()
911 		nfs_end_page_writeback(req);  in nfs_write_completion()
912 		nfs_release_request(req);  in nfs_write_completion()
929 	struct nfs_page *req, *tmp;  in nfs_scan_commit_list()  local
932 	list_for_each_entry_safe(req, tmp, src, wb_list) {  in nfs_scan_commit_list()
933 		if (!nfs_lock_request(req))  in nfs_scan_commit_list()
935 		kref_get(&req->wb_kref);  in nfs_scan_commit_list()
937 			list_safe_reset_next(req, tmp, wb_list);  in nfs_scan_commit_list()
938 		nfs_request_remove_commit_list(req, cinfo);  in nfs_scan_commit_list()
939 		nfs_list_add_request(req, dst);  in nfs_scan_commit_list()
986 	struct nfs_page *req;  in nfs_try_to_update_request()  local
998 		req = nfs_page_find_head_request_locked(NFS_I(inode), page);  in nfs_try_to_update_request()
999 		if (req == NULL)  in nfs_try_to_update_request()
1003 		WARN_ON_ONCE(req->wb_head != req);  in nfs_try_to_update_request()
1004 		WARN_ON_ONCE(req->wb_this_page != req);  in nfs_try_to_update_request()
1006 		rqend = req->wb_offset + req->wb_bytes;  in nfs_try_to_update_request()
1014 		    || end < req->wb_offset)  in nfs_try_to_update_request()
1017 		if (nfs_lock_request(req))  in nfs_try_to_update_request()
1022 		error = nfs_wait_on_request(req);  in nfs_try_to_update_request()
1023 		nfs_release_request(req);  in nfs_try_to_update_request()
1030 	if (offset < req->wb_offset) {  in nfs_try_to_update_request()
1031 		req->wb_offset = offset;  in nfs_try_to_update_request()
1032 		req->wb_pgbase = offset;  in nfs_try_to_update_request()
1035 		req->wb_bytes = end - req->wb_offset;  in nfs_try_to_update_request()
1037 		req->wb_bytes = rqend - req->wb_offset;  in nfs_try_to_update_request()
1039 	if (req)  in nfs_try_to_update_request()
1040 		nfs_clear_request_commit(req);  in nfs_try_to_update_request()
1042 	return req;  in nfs_try_to_update_request()
1045 	nfs_release_request(req);  in nfs_try_to_update_request()
1062 	struct nfs_page	*req;  in nfs_setup_write_request()  local
1064 	req = nfs_try_to_update_request(inode, page, offset, bytes);  in nfs_setup_write_request()
1065 	if (req != NULL)  in nfs_setup_write_request()
1067 	req = nfs_create_request(ctx, page, NULL, offset, bytes);  in nfs_setup_write_request()
1068 	if (IS_ERR(req))  in nfs_setup_write_request()
1070 	nfs_inode_add_request(inode, req);  in nfs_setup_write_request()
1072 	return req;  in nfs_setup_write_request()
1078 	struct nfs_page	*req;  in nfs_writepage_setup()  local
1080 	req = nfs_setup_write_request(ctx, page, offset, count);  in nfs_writepage_setup()
1081 	if (IS_ERR(req))  in nfs_writepage_setup()
1082 		return PTR_ERR(req);  in nfs_writepage_setup()
1085 	nfs_mark_uptodate(req);  in nfs_writepage_setup()
1086 	nfs_mark_request_dirty(req);  in nfs_writepage_setup()
1087 	nfs_unlock_and_release_request(req);  in nfs_writepage_setup()
1096 	struct nfs_page	*req;  in nfs_flush_incompatible()  local
1107 		req = nfs_page_find_head_request(page);  in nfs_flush_incompatible()
1108 		if (req == NULL)  in nfs_flush_incompatible()
1110 		l_ctx = req->wb_lock_context;  in nfs_flush_incompatible()
1111 		do_flush = req->wb_page != page || req->wb_context != ctx;  in nfs_flush_incompatible()
1113 		do_flush |= req->wb_this_page != req;  in nfs_flush_incompatible()
1120 		nfs_release_request(req);  in nfs_flush_incompatible()
1289 static void nfs_redirty_request(struct nfs_page *req)  in nfs_redirty_request()  argument
1291 	nfs_mark_request_dirty(req);  in nfs_redirty_request()
1292 	set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);  in nfs_redirty_request()
1293 	nfs_unlock_request(req);  in nfs_redirty_request()
1294 	nfs_end_page_writeback(req);  in nfs_redirty_request()
1295 	nfs_release_request(req);  in nfs_redirty_request()
1300 	struct nfs_page	*req;  in nfs_async_write_error()  local
1303 		req = nfs_list_entry(head->next);  in nfs_async_write_error()
1304 		nfs_list_remove_request(req);  in nfs_async_write_error()
1305 		nfs_redirty_request(req);  in nfs_async_write_error()
1583 	struct nfs_page *req;  in nfs_get_lwb()  local
1585 	list_for_each_entry(req, head, wb_list)  in nfs_get_lwb()
1586 		if (lwb < (req_offset(req) + req->wb_bytes))  in nfs_get_lwb()
1587 			lwb = req_offset(req) + req->wb_bytes;  in nfs_get_lwb()
1634 	struct nfs_page *req;  in nfs_retry_commit()  local
1637 		req = nfs_list_entry(page_list->next);  in nfs_retry_commit()
1638 		nfs_list_remove_request(req);  in nfs_retry_commit()
1639 		nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);  in nfs_retry_commit()
1641 			nfs_clear_page_commit(req->wb_page);  in nfs_retry_commit()
1642 		nfs_unlock_and_release_request(req);  in nfs_retry_commit()
1688 	struct nfs_page	*req;  in nfs_commit_release_pages()  local
1694 		req = nfs_list_entry(data->pages.next);  in nfs_commit_release_pages()
1695 		nfs_list_remove_request(req);  in nfs_commit_release_pages()
1696 		nfs_clear_page_commit(req->wb_page);  in nfs_commit_release_pages()
1699 			req->wb_context->dentry->d_sb->s_id,  in nfs_commit_release_pages()
1700 			(unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),  in nfs_commit_release_pages()
1701 			req->wb_bytes,  in nfs_commit_release_pages()
1702 			(long long)req_offset(req));  in nfs_commit_release_pages()
1704 			nfs_context_set_write_error(req->wb_context, status);  in nfs_commit_release_pages()
1705 			nfs_inode_remove_request(req);  in nfs_commit_release_pages()
1712 		if (!memcmp(&req->wb_verf, &data->verf.verifier, sizeof(req->wb_verf))) {  in nfs_commit_release_pages()
1714 			nfs_inode_remove_request(req);  in nfs_commit_release_pages()
1720 		nfs_mark_request_dirty(req);  in nfs_commit_release_pages()
1721 		set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);  in nfs_commit_release_pages()
1723 		nfs_unlock_and_release_request(req);  in nfs_commit_release_pages()
1871 	struct nfs_page *req;  in nfs_wb_page_cancel()  local
1878 	req = nfs_lock_and_join_requests(page, false);  in nfs_wb_page_cancel()
1880 	if (IS_ERR(req)) {  in nfs_wb_page_cancel()
1881 		ret = PTR_ERR(req);  in nfs_wb_page_cancel()
1882 	} else if (req) {  in nfs_wb_page_cancel()
1887 		nfs_inode_remove_request(req);  in nfs_wb_page_cancel()
1888 		nfs_unlock_and_release_request(req);  in nfs_wb_page_cancel()