61 	hdr->req = nfs_list_entry(mirror->pg_list.next);  in nfs_pgheader_init()
63 	hdr->cred = hdr->req->wb_context->cred;  in nfs_pgheader_init()
64 	hdr->io_start = req_offset(hdr->req);  in nfs_pgheader_init()
168 nfs_page_group_lock(struct nfs_page *req, bool nonblock)  in nfs_page_group_lock()  argument
170 	struct nfs_page *head = req->wb_head;  in nfs_page_group_lock()
191 nfs_page_group_lock_wait(struct nfs_page *req)  in nfs_page_group_lock_wait()  argument
193 	struct nfs_page *head = req->wb_head;  in nfs_page_group_lock_wait()
206 nfs_page_group_unlock(struct nfs_page *req)  in nfs_page_group_unlock()  argument
208 	struct nfs_page *head = req->wb_head;  in nfs_page_group_unlock()
224 nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit)  in nfs_page_group_sync_on_bit_locked()  argument
226 	struct nfs_page *head = req->wb_head;  in nfs_page_group_sync_on_bit_locked()
230 	WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags));  in nfs_page_group_sync_on_bit_locked()
232 	tmp = req->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
233 	while (tmp != req) {  in nfs_page_group_sync_on_bit_locked()
240 	tmp = req;  in nfs_page_group_sync_on_bit_locked()
244 	} while (tmp != req);  in nfs_page_group_sync_on_bit_locked()
255 bool nfs_page_group_sync_on_bit(struct nfs_page *req, unsigned int bit)  in nfs_page_group_sync_on_bit()  argument
259 	nfs_page_group_lock(req, false);  in nfs_page_group_sync_on_bit()
260 	ret = nfs_page_group_sync_on_bit_locked(req, bit);  in nfs_page_group_sync_on_bit()
261 	nfs_page_group_unlock(req);  in nfs_page_group_sync_on_bit()
273 nfs_page_group_init(struct nfs_page *req, struct nfs_page *prev)  in nfs_page_group_init()  argument
276 	WARN_ON_ONCE(prev == req);  in nfs_page_group_init()
280 		req->wb_head = req;  in nfs_page_group_init()
281 		req->wb_this_page = req;  in nfs_page_group_init()
286 		req->wb_head = prev->wb_head;  in nfs_page_group_init()
287 		req->wb_this_page = prev->wb_this_page;  in nfs_page_group_init()
288 		prev->wb_this_page = req;  in nfs_page_group_init()
292 		kref_get(&req->wb_head->wb_kref);  in nfs_page_group_init()
298 			inode = page_file_mapping(req->wb_page)->host;  in nfs_page_group_init()
299 			set_bit(PG_INODE_REF, &req->wb_flags);  in nfs_page_group_init()
300 			kref_get(&req->wb_kref);  in nfs_page_group_init()
318 	struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref);  in nfs_page_group_destroy()  local
322 	if (req->wb_head != req)  in nfs_page_group_destroy()
323 		nfs_release_request(req->wb_head);  in nfs_page_group_destroy()
325 	if (!nfs_page_group_sync_on_bit(req, PG_TEARDOWN))  in nfs_page_group_destroy()
328 	tmp = req;  in nfs_page_group_destroy()
336 	} while (tmp != req);  in nfs_page_group_destroy()
356 	struct nfs_page		*req;  in nfs_create_request()  local
362 	req = nfs_page_alloc();  in nfs_create_request()
363 	if (req == NULL)  in nfs_create_request()
369 		nfs_page_free(req);  in nfs_create_request()
372 	req->wb_lock_context = l_ctx;  in nfs_create_request()
378 	req->wb_page    = page;  in nfs_create_request()
379 	req->wb_index	= page_file_index(page);  in nfs_create_request()
381 	req->wb_offset  = offset;  in nfs_create_request()
382 	req->wb_pgbase	= offset;  in nfs_create_request()
383 	req->wb_bytes   = count;  in nfs_create_request()
384 	req->wb_context = get_nfs_open_context(ctx);  in nfs_create_request()
385 	kref_init(&req->wb_kref);  in nfs_create_request()
386 	nfs_page_group_init(req, last);  in nfs_create_request()
387 	return req;  in nfs_create_request()
394 void nfs_unlock_request(struct nfs_page *req)  in nfs_unlock_request()  argument
396 	if (!NFS_WBACK_BUSY(req)) {  in nfs_unlock_request()
401 	clear_bit(PG_BUSY, &req->wb_flags);  in nfs_unlock_request()
403 	wake_up_bit(&req->wb_flags, PG_BUSY);  in nfs_unlock_request()
410 void nfs_unlock_and_release_request(struct nfs_page *req)  in nfs_unlock_and_release_request()  argument
412 	nfs_unlock_request(req);  in nfs_unlock_and_release_request()
413 	nfs_release_request(req);  in nfs_unlock_and_release_request()
423 static void nfs_clear_request(struct nfs_page *req)  in nfs_clear_request()  argument
425 	struct page *page = req->wb_page;  in nfs_clear_request()
426 	struct nfs_open_context *ctx = req->wb_context;  in nfs_clear_request()
427 	struct nfs_lock_context *l_ctx = req->wb_lock_context;  in nfs_clear_request()
431 		req->wb_page = NULL;  in nfs_clear_request()
436 		req->wb_lock_context = NULL;  in nfs_clear_request()
440 		req->wb_context = NULL;  in nfs_clear_request()
450 void nfs_free_request(struct nfs_page *req)  in nfs_free_request()  argument
452 	WARN_ON_ONCE(req->wb_this_page != req);  in nfs_free_request()
455 	WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));  in nfs_free_request()
456 	WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags));  in nfs_free_request()
457 	WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags));  in nfs_free_request()
458 	WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags));  in nfs_free_request()
459 	WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags));  in nfs_free_request()
462 	nfs_clear_request(req);  in nfs_free_request()
463 	nfs_page_free(req);  in nfs_free_request()
466 void nfs_release_request(struct nfs_page *req)  in nfs_release_request()  argument
468 	kref_put(&req->wb_kref, nfs_page_group_destroy);  in nfs_release_request()
479 nfs_wait_on_request(struct nfs_page *req)  in nfs_wait_on_request()  argument
481 	return wait_on_bit_io(&req->wb_flags, PG_BUSY,  in nfs_wait_on_request()
495 			   struct nfs_page *prev, struct nfs_page *req)  in nfs_generic_pg_test()  argument
510 	if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *  in nfs_generic_pg_test()
514 	return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);  in nfs_generic_pg_test()
570 	struct nfs_page *req = hdr->req;  in nfs_pgio_rpcsetup()  local
576 	hdr->args.offset = req_offset(req) + offset;  in nfs_pgio_rpcsetup()
579 	hdr->args.pgbase = req->wb_pgbase + offset;  in nfs_pgio_rpcsetup()
582 	hdr->args.context = get_nfs_open_context(req->wb_context);  in nfs_pgio_rpcsetup()
583 	hdr->args.lock_context = req->wb_lock_context;  in nfs_pgio_rpcsetup()
796 	struct nfs_page		*req;  in nfs_generic_pgio()  local
812 		req = nfs_list_entry(head->next);  in nfs_generic_pgio()
813 		nfs_list_remove_request(req);  in nfs_generic_pgio()
814 		nfs_list_add_request(req, &hdr->pages);  in nfs_generic_pgio()
816 		if (!last_page || last_page != req->wb_page) {  in nfs_generic_pgio()
820 			*pages++ = last_page = req->wb_page;  in nfs_generic_pgio()
869 				       struct nfs_page *req)  in nfs_pageio_setup_mirroring()  argument
876 	mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);  in nfs_pageio_setup_mirroring()
932 				      struct nfs_page *req,  in nfs_can_coalesce_requests()  argument
939 		if (!nfs_match_open_context(req->wb_context, prev->wb_context))  in nfs_can_coalesce_requests()
941 		flctx = d_inode(req->wb_context->dentry)->i_flctx;  in nfs_can_coalesce_requests()
945 		    !nfs_match_lock_context(req->wb_lock_context,  in nfs_can_coalesce_requests()
948 		if (req_offset(req) != req_offset(prev) + prev->wb_bytes)  in nfs_can_coalesce_requests()
950 		if (req->wb_page == prev->wb_page) {  in nfs_can_coalesce_requests()
951 			if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes)  in nfs_can_coalesce_requests()
954 			if (req->wb_pgbase != 0 ||  in nfs_can_coalesce_requests()
959 	size = pgio->pg_ops->pg_test(pgio, prev, req);  in nfs_can_coalesce_requests()
960 	WARN_ON_ONCE(size > req->wb_bytes);  in nfs_can_coalesce_requests()
961 	if (size && size < req->wb_bytes)  in nfs_can_coalesce_requests()
962 		req->wb_bytes = size;  in nfs_can_coalesce_requests()
975 				     struct nfs_page *req)  in nfs_pageio_do_add_request()  argument
985 			desc->pg_ops->pg_init(desc, req);  in nfs_pageio_do_add_request()
986 		mirror->pg_base = req->wb_pgbase;  in nfs_pageio_do_add_request()
988 	if (!nfs_can_coalesce_requests(prev, req, desc))  in nfs_pageio_do_add_request()
990 	nfs_list_remove_request(req);  in nfs_pageio_do_add_request()
991 	nfs_list_add_request(req, &mirror->pg_list);  in nfs_pageio_do_add_request()
992 	mirror->pg_count += req->wb_bytes;  in nfs_pageio_do_add_request()
1029 			   struct nfs_page *req)  in __nfs_pageio_add_request()  argument
1037 	nfs_page_group_lock(req, false);  in __nfs_pageio_add_request()
1039 	subreq = req;  in __nfs_pageio_add_request()
1051 			nfs_page_group_unlock(req);  in __nfs_pageio_add_request()
1059 			nfs_page_group_lock(req, false);  in __nfs_pageio_add_request()
1073 			subreq = nfs_create_request(req->wb_context,  in __nfs_pageio_add_request()
1074 					req->wb_page,  in __nfs_pageio_add_request()
1080 			subreq->wb_index = req->wb_index;  in __nfs_pageio_add_request()
1084 	nfs_page_group_unlock(req);  in __nfs_pageio_add_request()
1088 	nfs_page_group_unlock(req);  in __nfs_pageio_add_request()
1107 			struct nfs_page *req;  in nfs_do_recoalesce()  local
1109 			req = list_first_entry(&head, struct nfs_page, wb_list);  in nfs_do_recoalesce()
1110 			nfs_list_remove_request(req);  in nfs_do_recoalesce()
1111 			if (__nfs_pageio_add_request(desc, req))  in nfs_do_recoalesce()
1125 		struct nfs_page *req)  in nfs_pageio_add_request_mirror()  argument
1130 		ret = __nfs_pageio_add_request(desc, req);  in nfs_pageio_add_request_mirror()
1142 			   struct nfs_page *req)  in nfs_pageio_add_request()  argument
1148 	pgbase = req->wb_pgbase;  in nfs_pageio_add_request()
1149 	offset = req->wb_offset;  in nfs_pageio_add_request()
1150 	bytes = req->wb_bytes;  in nfs_pageio_add_request()
1152 	nfs_pageio_setup_mirroring(desc, req);  in nfs_pageio_add_request()
1156 			nfs_page_group_lock(req, false);  in nfs_pageio_add_request()
1159 			for (lastreq = req->wb_head;  in nfs_pageio_add_request()
1160 			     lastreq->wb_this_page != req->wb_head;  in nfs_pageio_add_request()
1164 			dupreq = nfs_create_request(req->wb_context,  in nfs_pageio_add_request()
1165 					req->wb_page, lastreq, pgbase, bytes);  in nfs_pageio_add_request()
1168 				nfs_page_group_unlock(req);  in nfs_pageio_add_request()
1173 			nfs_page_group_unlock(req);  in nfs_pageio_add_request()
1175 			dupreq->wb_index = req->wb_index;  in nfs_pageio_add_request()
1177 			dupreq = req;  in nfs_pageio_add_request()
1228 		struct nfs_page *req = nfs_list_entry(hdr->pages.next);  in nfs_pageio_resend()  local
1230 		nfs_list_remove_request(req);  in nfs_pageio_resend()
1231 		if (!nfs_pageio_add_request(desc, req))  in nfs_pageio_resend()
1232 			nfs_list_add_request(req, &failed);  in nfs_pageio_resend()