609 static void ff_layout_reset_write(struct nfs_pgio_header *hdr, bool retry_pnfs)  in ff_layout_reset_write()  argument
611 	struct rpc_task *task = &hdr->task;  in ff_layout_reset_write()
613 	pnfs_layoutcommit_inode(hdr->inode, false);  in ff_layout_reset_write()
618 			hdr->task.tk_pid,  in ff_layout_reset_write()
619 			hdr->inode->i_sb->s_id,  in ff_layout_reset_write()
620 			(unsigned long long)NFS_FILEID(hdr->inode),  in ff_layout_reset_write()
621 			hdr->args.count,  in ff_layout_reset_write()
622 			(unsigned long long)hdr->args.offset);  in ff_layout_reset_write()
624 		if (!hdr->dreq) {  in ff_layout_reset_write()
627 			ctx = nfs_list_entry(hdr->pages.next)->wb_context;  in ff_layout_reset_write()
629 			hdr->completion_ops->error_cleanup(&hdr->pages);  in ff_layout_reset_write()
631 			nfs_direct_set_resched_writes(hdr->dreq);  in ff_layout_reset_write()
633 			hdr->verf.committed = NFS_UNSTABLE;  in ff_layout_reset_write()
634 			hdr->good_bytes = hdr->args.count;  in ff_layout_reset_write()
639 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {  in ff_layout_reset_write()
642 			hdr->task.tk_pid,  in ff_layout_reset_write()
643 			hdr->inode->i_sb->s_id,  in ff_layout_reset_write()
644 			(unsigned long long)NFS_FILEID(hdr->inode),  in ff_layout_reset_write()
645 			hdr->args.count,  in ff_layout_reset_write()
646 			(unsigned long long)hdr->args.offset);  in ff_layout_reset_write()
648 		task->tk_status = pnfs_write_done_resend_to_mds(hdr);  in ff_layout_reset_write()
652 static void ff_layout_reset_read(struct nfs_pgio_header *hdr)  in ff_layout_reset_read()  argument
654 	struct rpc_task *task = &hdr->task;  in ff_layout_reset_read()
656 	pnfs_layoutcommit_inode(hdr->inode, false);  in ff_layout_reset_read()
658 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {  in ff_layout_reset_read()
661 			hdr->task.tk_pid,  in ff_layout_reset_read()
662 			hdr->inode->i_sb->s_id,  in ff_layout_reset_read()
663 			(unsigned long long)NFS_FILEID(hdr->inode),  in ff_layout_reset_read()
664 			hdr->args.count,  in ff_layout_reset_read()
665 			(unsigned long long)hdr->args.offset);  in ff_layout_reset_read()
667 		task->tk_status = pnfs_read_done_resend_to_mds(hdr);  in ff_layout_reset_read()
847 				struct nfs_pgio_header *hdr)  in ff_layout_read_done_cb()  argument
852 	trace_nfs4_pnfs_read(hdr, task->tk_status);  in ff_layout_read_done_cb()
853 	if (task->tk_status == -ETIMEDOUT && !hdr->res.op_status)  in ff_layout_read_done_cb()
854 		hdr->res.op_status = NFS4ERR_NXIO;  in ff_layout_read_done_cb()
855 	if (task->tk_status < 0 && hdr->res.op_status)  in ff_layout_read_done_cb()
856 		ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx,  in ff_layout_read_done_cb()
857 					    hdr->args.offset, hdr->args.count,  in ff_layout_read_done_cb()
858 					    hdr->res.op_status, OP_READ);  in ff_layout_read_done_cb()
859 	err = ff_layout_async_handle_error(task, hdr->args.context->state,  in ff_layout_read_done_cb()
860 					   hdr->ds_clp, hdr->lseg,  in ff_layout_read_done_cb()
861 					   hdr->pgio_mirror_idx);  in ff_layout_read_done_cb()
866 			&hdr->lseg->pls_layout->plh_flags);  in ff_layout_read_done_cb()
867 		pnfs_read_resend_pnfs(hdr);  in ff_layout_read_done_cb()
870 		inode = hdr->lseg->pls_layout->plh_inode;  in ff_layout_read_done_cb()
871 		pnfs_error_mark_layout_for_return(inode, hdr->lseg);  in ff_layout_read_done_cb()
872 		ff_layout_reset_read(hdr);  in ff_layout_read_done_cb()
892 ff_layout_set_layoutcommit(struct nfs_pgio_header *hdr)  in ff_layout_set_layoutcommit()  argument
894 	pnfs_set_layoutcommit(hdr->inode, hdr->lseg,  in ff_layout_set_layoutcommit()
895 			hdr->mds_offset + hdr->res.count);  in ff_layout_set_layoutcommit()
896 	dprintk("%s inode %lu pls_end_pos %lu\n", __func__, hdr->inode->i_ino,  in ff_layout_set_layoutcommit()
897 		(unsigned long) NFS_I(hdr->inode)->layout->plh_lwb);  in ff_layout_set_layoutcommit()
910 					 struct nfs_pgio_header *hdr)  in ff_layout_read_prepare_common()  argument
912 	if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) {  in ff_layout_read_prepare_common()
916 	if (ff_layout_reset_to_mds(hdr->lseg, hdr->pgio_mirror_idx)) {  in ff_layout_read_prepare_common()
918 		if (ff_layout_has_available_ds(hdr->lseg))  in ff_layout_read_prepare_common()
919 			pnfs_read_resend_pnfs(hdr);  in ff_layout_read_prepare_common()
921 			ff_layout_reset_read(hdr);  in ff_layout_read_prepare_common()
925 	hdr->pgio_done_cb = ff_layout_read_done_cb;  in ff_layout_read_prepare_common()
937 	struct nfs_pgio_header *hdr = data;  in ff_layout_read_prepare_v3()  local
939 	if (ff_layout_read_prepare_common(task, hdr))  in ff_layout_read_prepare_v3()
963 	struct nfs_pgio_header *hdr = data;  in ff_layout_read_prepare_v4()  local
965 	if (ff_layout_read_prepare_common(task, hdr))  in ff_layout_read_prepare_v4()
968 	if (ff_layout_setup_sequence(hdr->ds_clp,  in ff_layout_read_prepare_v4()
969 				     &hdr->args.seq_args,  in ff_layout_read_prepare_v4()
970 				     &hdr->res.seq_res,  in ff_layout_read_prepare_v4()
974 	if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context,  in ff_layout_read_prepare_v4()
975 			hdr->args.lock_context, FMODE_READ) == -EIO)  in ff_layout_read_prepare_v4()
981 	struct nfs_pgio_header *hdr = data;  in ff_layout_read_call_done()  local
985 	if (test_bit(NFS_IOHDR_REDO, &hdr->flags) &&  in ff_layout_read_call_done()
987 		nfs4_sequence_done(task, &hdr->res.seq_res);  in ff_layout_read_call_done()
992 	hdr->mds_ops->rpc_call_done(task, hdr);  in ff_layout_read_call_done()
997 	struct nfs_pgio_header *hdr = data;  in ff_layout_read_count_stats()  local
1000 	    &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_READ]);  in ff_layout_read_count_stats()
1004 				struct nfs_pgio_header *hdr)  in ff_layout_write_done_cb()  argument
1009 	trace_nfs4_pnfs_write(hdr, task->tk_status);  in ff_layout_write_done_cb()
1010 	if (task->tk_status == -ETIMEDOUT && !hdr->res.op_status)  in ff_layout_write_done_cb()
1011 		hdr->res.op_status = NFS4ERR_NXIO;  in ff_layout_write_done_cb()
1012 	if (task->tk_status < 0 && hdr->res.op_status)  in ff_layout_write_done_cb()
1013 		ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx,  in ff_layout_write_done_cb()
1014 					    hdr->args.offset, hdr->args.count,  in ff_layout_write_done_cb()
1015 					    hdr->res.op_status, OP_WRITE);  in ff_layout_write_done_cb()
1016 	err = ff_layout_async_handle_error(task, hdr->args.context->state,  in ff_layout_write_done_cb()
1017 					   hdr->ds_clp, hdr->lseg,  in ff_layout_write_done_cb()
1018 					   hdr->pgio_mirror_idx);  in ff_layout_write_done_cb()
1023 		inode = hdr->lseg->pls_layout->plh_inode;  in ff_layout_write_done_cb()
1024 		pnfs_error_mark_layout_for_return(inode, hdr->lseg);  in ff_layout_write_done_cb()
1026 			pnfs_set_retry_layoutget(hdr->lseg->pls_layout);  in ff_layout_write_done_cb()
1027 			ff_layout_reset_write(hdr, true);  in ff_layout_write_done_cb()
1029 			pnfs_clear_retry_layoutget(hdr->lseg->pls_layout);  in ff_layout_write_done_cb()
1030 			ff_layout_reset_write(hdr, false);  in ff_layout_write_done_cb()
1038 	if (hdr->res.verf->committed == NFS_FILE_SYNC ||  in ff_layout_write_done_cb()
1039 	    hdr->res.verf->committed == NFS_DATA_SYNC)  in ff_layout_write_done_cb()
1040 		ff_layout_set_layoutcommit(hdr);  in ff_layout_write_done_cb()
1043 	hdr->fattr.valid = 0;  in ff_layout_write_done_cb()
1045 		nfs_writeback_update_inode(hdr);  in ff_layout_write_done_cb()
1089 					  struct nfs_pgio_header *hdr)  in ff_layout_write_prepare_common()  argument
1091 	if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) {  in ff_layout_write_prepare_common()
1096 	if (ff_layout_reset_to_mds(hdr->lseg, hdr->pgio_mirror_idx)) {  in ff_layout_write_prepare_common()
1099 		retry_pnfs = ff_layout_has_available_ds(hdr->lseg);  in ff_layout_write_prepare_common()
1102 		ff_layout_reset_write(hdr, retry_pnfs);  in ff_layout_write_prepare_common()
1112 	struct nfs_pgio_header *hdr = data;  in ff_layout_write_prepare_v3()  local
1114 	if (ff_layout_write_prepare_common(task, hdr))  in ff_layout_write_prepare_v3()
1122 	struct nfs_pgio_header *hdr = data;  in ff_layout_write_prepare_v4()  local
1124 	if (ff_layout_write_prepare_common(task, hdr))  in ff_layout_write_prepare_v4()
1127 	if (ff_layout_setup_sequence(hdr->ds_clp,  in ff_layout_write_prepare_v4()
1128 				     &hdr->args.seq_args,  in ff_layout_write_prepare_v4()
1129 				     &hdr->res.seq_res,  in ff_layout_write_prepare_v4()
1133 	if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context,  in ff_layout_write_prepare_v4()
1134 			hdr->args.lock_context, FMODE_WRITE) == -EIO)  in ff_layout_write_prepare_v4()
1140 	struct nfs_pgio_header *hdr = data;  in ff_layout_write_call_done()  local
1142 	if (test_bit(NFS_IOHDR_REDO, &hdr->flags) &&  in ff_layout_write_call_done()
1144 		nfs4_sequence_done(task, &hdr->res.seq_res);  in ff_layout_write_call_done()
1149 	hdr->mds_ops->rpc_call_done(task, hdr);  in ff_layout_write_call_done()
1154 	struct nfs_pgio_header *hdr = data;  in ff_layout_write_count_stats()  local
1157 	    &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_WRITE]);  in ff_layout_write_count_stats()
1226 ff_layout_read_pagelist(struct nfs_pgio_header *hdr)  in ff_layout_read_pagelist()  argument
1228 	struct pnfs_layout_segment *lseg = hdr->lseg;  in ff_layout_read_pagelist()
1232 	loff_t offset = hdr->args.offset;  in ff_layout_read_pagelist()
1233 	u32 idx = hdr->pgio_mirror_idx;  in ff_layout_read_pagelist()
1238 		__func__, hdr->inode->i_ino,  in ff_layout_read_pagelist()
1239 		hdr->args.pgbase, (size_t)hdr->args.count, offset);  in ff_layout_read_pagelist()
1246 						   hdr->inode);  in ff_layout_read_pagelist()
1250 	ds_cred = ff_layout_get_ds_cred(lseg, idx, hdr->cred);  in ff_layout_read_pagelist()
1260 	hdr->ds_clp = ds->ds_clp;  in ff_layout_read_pagelist()
1263 		hdr->args.fh = fh;  in ff_layout_read_pagelist()
1269 	hdr->args.offset = offset;  in ff_layout_read_pagelist()
1270 	hdr->mds_offset = offset;  in ff_layout_read_pagelist()
1273 	nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,  in ff_layout_read_pagelist()
1288 ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)  in ff_layout_write_pagelist()  argument
1290 	struct pnfs_layout_segment *lseg = hdr->lseg;  in ff_layout_write_pagelist()
1294 	loff_t offset = hdr->args.offset;  in ff_layout_write_pagelist()
1297 	int idx = hdr->pgio_mirror_idx;  in ff_layout_write_pagelist()
1304 						   hdr->inode);  in ff_layout_write_pagelist()
1308 	ds_cred = ff_layout_get_ds_cred(lseg, idx, hdr->cred);  in ff_layout_write_pagelist()
1315 		__func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count,  in ff_layout_write_pagelist()
1319 	hdr->pgio_done_cb = ff_layout_write_done_cb;  in ff_layout_write_pagelist()
1321 	hdr->ds_clp = ds->ds_clp;  in ff_layout_write_pagelist()
1322 	hdr->ds_commit_idx = idx;  in ff_layout_write_pagelist()
1325 		hdr->args.fh = fh;  in ff_layout_write_pagelist()
1331 	hdr->args.offset = offset;  in ff_layout_write_pagelist()
1334 	nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,  in ff_layout_write_pagelist()
1433 	struct pnfs_layout_hdr *hdr = &flo->generic_hdr;  in ff_layout_encode_ioerr()  local
1442 	spin_lock(&hdr->plh_inode->i_lock);  in ff_layout_encode_ioerr()
1444 	spin_unlock(&hdr->plh_inode->i_lock);  in ff_layout_encode_ioerr()