900 static void ff_layout_reset_write(struct nfs_pgio_header *hdr, bool retry_pnfs)  in ff_layout_reset_write()  argument
902 	struct rpc_task *task = &hdr->task;  in ff_layout_reset_write()
904 	pnfs_layoutcommit_inode(hdr->inode, false);  in ff_layout_reset_write()
909 			hdr->task.tk_pid,  in ff_layout_reset_write()
910 			hdr->inode->i_sb->s_id,  in ff_layout_reset_write()
911 			(unsigned long long)NFS_FILEID(hdr->inode),  in ff_layout_reset_write()
912 			hdr->args.count,  in ff_layout_reset_write()
913 			(unsigned long long)hdr->args.offset);  in ff_layout_reset_write()
915 		if (!hdr->dreq) {  in ff_layout_reset_write()
918 			ctx = nfs_list_entry(hdr->pages.next)->wb_context;  in ff_layout_reset_write()
920 			hdr->completion_ops->error_cleanup(&hdr->pages);  in ff_layout_reset_write()
922 			nfs_direct_set_resched_writes(hdr->dreq);  in ff_layout_reset_write()
924 			hdr->verf.committed = NFS_UNSTABLE;  in ff_layout_reset_write()
925 			hdr->good_bytes = hdr->args.count;  in ff_layout_reset_write()
930 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {  in ff_layout_reset_write()
933 			hdr->task.tk_pid,  in ff_layout_reset_write()
934 			hdr->inode->i_sb->s_id,  in ff_layout_reset_write()
935 			(unsigned long long)NFS_FILEID(hdr->inode),  in ff_layout_reset_write()
936 			hdr->args.count,  in ff_layout_reset_write()
937 			(unsigned long long)hdr->args.offset);  in ff_layout_reset_write()
939 		task->tk_status = pnfs_write_done_resend_to_mds(hdr);  in ff_layout_reset_write()
943 static void ff_layout_reset_read(struct nfs_pgio_header *hdr)  in ff_layout_reset_read()  argument
945 	struct rpc_task *task = &hdr->task;  in ff_layout_reset_read()
947 	pnfs_layoutcommit_inode(hdr->inode, false);  in ff_layout_reset_read()
949 	if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {  in ff_layout_reset_read()
952 			hdr->task.tk_pid,  in ff_layout_reset_read()
953 			hdr->inode->i_sb->s_id,  in ff_layout_reset_read()
954 			(unsigned long long)NFS_FILEID(hdr->inode),  in ff_layout_reset_read()
955 			hdr->args.count,  in ff_layout_reset_read()
956 			(unsigned long long)hdr->args.offset);  in ff_layout_reset_read()
958 		task->tk_status = pnfs_read_done_resend_to_mds(hdr);  in ff_layout_reset_read()
1172 				struct nfs_pgio_header *hdr)  in ff_layout_read_done_cb()  argument
1176 	trace_nfs4_pnfs_read(hdr, task->tk_status);  in ff_layout_read_done_cb()
1178 		ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx,  in ff_layout_read_done_cb()
1179 					    hdr->args.offset, hdr->args.count,  in ff_layout_read_done_cb()
1180 					    hdr->res.op_status, OP_READ,  in ff_layout_read_done_cb()
1182 	err = ff_layout_async_handle_error(task, hdr->args.context->state,  in ff_layout_read_done_cb()
1183 					   hdr->ds_clp, hdr->lseg,  in ff_layout_read_done_cb()
1184 					   hdr->pgio_mirror_idx);  in ff_layout_read_done_cb()
1188 		if (ff_layout_choose_best_ds_for_read(hdr->lseg,  in ff_layout_read_done_cb()
1189 					hdr->pgio_mirror_idx + 1,  in ff_layout_read_done_cb()
1190 					&hdr->pgio_mirror_idx))  in ff_layout_read_done_cb()
1193 			&hdr->lseg->pls_layout->plh_flags);  in ff_layout_read_done_cb()
1194 		pnfs_read_resend_pnfs(hdr);  in ff_layout_read_done_cb()
1197 		ff_layout_reset_read(hdr);  in ff_layout_read_done_cb()
1225 ff_layout_set_layoutcommit(struct nfs_pgio_header *hdr)  in ff_layout_set_layoutcommit()  argument
1227 	if (!ff_layout_need_layoutcommit(hdr->lseg))  in ff_layout_set_layoutcommit()
1230 	pnfs_set_layoutcommit(hdr->inode, hdr->lseg,  in ff_layout_set_layoutcommit()
1231 			hdr->mds_offset + hdr->res.count);  in ff_layout_set_layoutcommit()
1232 	dprintk("%s inode %lu pls_end_pos %lu\n", __func__, hdr->inode->i_ino,  in ff_layout_set_layoutcommit()
1233 		(unsigned long) NFS_I(hdr->inode)->layout->plh_lwb);  in ff_layout_set_layoutcommit()
1246 					 struct nfs_pgio_header *hdr)  in ff_layout_read_prepare_common()  argument
1248 	nfs4_ff_layout_stat_io_start_read(hdr->inode,  in ff_layout_read_prepare_common()
1249 			FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx),  in ff_layout_read_prepare_common()
1250 			hdr->args.count,  in ff_layout_read_prepare_common()
1253 	if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) {  in ff_layout_read_prepare_common()
1257 	if (ff_layout_reset_to_mds(hdr->lseg, hdr->pgio_mirror_idx)) {  in ff_layout_read_prepare_common()
1259 		if (ff_layout_has_available_ds(hdr->lseg))  in ff_layout_read_prepare_common()
1260 			pnfs_read_resend_pnfs(hdr);  in ff_layout_read_prepare_common()
1262 			ff_layout_reset_read(hdr);  in ff_layout_read_prepare_common()
1266 	hdr->pgio_done_cb = ff_layout_read_done_cb;  in ff_layout_read_prepare_common()
1278 	struct nfs_pgio_header *hdr = data;  in ff_layout_read_prepare_v3()  local
1280 	if (ff_layout_read_prepare_common(task, hdr))  in ff_layout_read_prepare_v3()
1304 	struct nfs_pgio_header *hdr = data;  in ff_layout_read_prepare_v4()  local
1306 	if (ff_layout_setup_sequence(hdr->ds_clp,  in ff_layout_read_prepare_v4()
1307 				     &hdr->args.seq_args,  in ff_layout_read_prepare_v4()
1308 				     &hdr->res.seq_res,  in ff_layout_read_prepare_v4()
1312 	if (ff_layout_read_prepare_common(task, hdr))  in ff_layout_read_prepare_v4()
1315 	if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context,  in ff_layout_read_prepare_v4()
1316 			hdr->args.lock_context, FMODE_READ) == -EIO)  in ff_layout_read_prepare_v4()
1322 	struct nfs_pgio_header *hdr = data;  in ff_layout_read_call_done()  local
1327 			FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx),  in ff_layout_read_call_done()
1328 			hdr->args.count, hdr->res.count);  in ff_layout_read_call_done()
1330 	if (test_bit(NFS_IOHDR_REDO, &hdr->flags) &&  in ff_layout_read_call_done()
1332 		nfs4_sequence_done(task, &hdr->res.seq_res);  in ff_layout_read_call_done()
1337 	hdr->mds_ops->rpc_call_done(task, hdr);  in ff_layout_read_call_done()
1342 	struct nfs_pgio_header *hdr = data;  in ff_layout_read_count_stats()  local
1345 	    &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_READ]);  in ff_layout_read_count_stats()
1349 				struct nfs_pgio_header *hdr)  in ff_layout_write_done_cb()  argument
1353 	trace_nfs4_pnfs_write(hdr, task->tk_status);  in ff_layout_write_done_cb()
1355 		ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx,  in ff_layout_write_done_cb()
1356 					    hdr->args.offset, hdr->args.count,  in ff_layout_write_done_cb()
1357 					    hdr->res.op_status, OP_WRITE,  in ff_layout_write_done_cb()
1359 	err = ff_layout_async_handle_error(task, hdr->args.context->state,  in ff_layout_write_done_cb()
1360 					   hdr->ds_clp, hdr->lseg,  in ff_layout_write_done_cb()
1361 					   hdr->pgio_mirror_idx);  in ff_layout_write_done_cb()
1365 		pnfs_set_retry_layoutget(hdr->lseg->pls_layout);  in ff_layout_write_done_cb()
1366 		ff_layout_reset_write(hdr, true);  in ff_layout_write_done_cb()
1369 		pnfs_clear_retry_layoutget(hdr->lseg->pls_layout);  in ff_layout_write_done_cb()
1370 		ff_layout_reset_write(hdr, false);  in ff_layout_write_done_cb()
1377 	if (hdr->res.verf->committed == NFS_FILE_SYNC ||  in ff_layout_write_done_cb()
1378 	    hdr->res.verf->committed == NFS_DATA_SYNC)  in ff_layout_write_done_cb()
1379 		ff_layout_set_layoutcommit(hdr);  in ff_layout_write_done_cb()
1382 	hdr->fattr.valid = 0;  in ff_layout_write_done_cb()
1384 		nfs_writeback_update_inode(hdr);  in ff_layout_write_done_cb()
1425 					  struct nfs_pgio_header *hdr)  in ff_layout_write_prepare_common()  argument
1427 	nfs4_ff_layout_stat_io_start_write(hdr->inode,  in ff_layout_write_prepare_common()
1428 			FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx),  in ff_layout_write_prepare_common()
1429 			hdr->args.count,  in ff_layout_write_prepare_common()
1432 	if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) {  in ff_layout_write_prepare_common()
1437 	if (ff_layout_reset_to_mds(hdr->lseg, hdr->pgio_mirror_idx)) {  in ff_layout_write_prepare_common()
1440 		retry_pnfs = ff_layout_has_available_ds(hdr->lseg);  in ff_layout_write_prepare_common()
1443 		ff_layout_reset_write(hdr, retry_pnfs);  in ff_layout_write_prepare_common()
1453 	struct nfs_pgio_header *hdr = data;  in ff_layout_write_prepare_v3()  local
1455 	if (ff_layout_write_prepare_common(task, hdr))  in ff_layout_write_prepare_v3()
1463 	struct nfs_pgio_header *hdr = data;  in ff_layout_write_prepare_v4()  local
1465 	if (ff_layout_setup_sequence(hdr->ds_clp,  in ff_layout_write_prepare_v4()
1466 				     &hdr->args.seq_args,  in ff_layout_write_prepare_v4()
1467 				     &hdr->res.seq_res,  in ff_layout_write_prepare_v4()
1471 	if (ff_layout_write_prepare_common(task, hdr))  in ff_layout_write_prepare_v4()
1474 	if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context,  in ff_layout_write_prepare_v4()
1475 			hdr->args.lock_context, FMODE_WRITE) == -EIO)  in ff_layout_write_prepare_v4()
1481 	struct nfs_pgio_header *hdr = data;  in ff_layout_write_call_done()  local
1484 			FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx),  in ff_layout_write_call_done()
1485 			hdr->args.count, hdr->res.count,  in ff_layout_write_call_done()
1486 			hdr->res.verf->committed);  in ff_layout_write_call_done()
1488 	if (test_bit(NFS_IOHDR_REDO, &hdr->flags) &&  in ff_layout_write_call_done()
1490 		nfs4_sequence_done(task, &hdr->res.seq_res);  in ff_layout_write_call_done()
1495 	hdr->mds_ops->rpc_call_done(task, hdr);  in ff_layout_write_call_done()
1500 	struct nfs_pgio_header *hdr = data;  in ff_layout_write_count_stats()  local
1503 	    &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_WRITE]);  in ff_layout_write_count_stats()
1601 ff_layout_read_pagelist(struct nfs_pgio_header *hdr)  in ff_layout_read_pagelist()  argument
1603 	struct pnfs_layout_segment *lseg = hdr->lseg;  in ff_layout_read_pagelist()
1607 	loff_t offset = hdr->args.offset;  in ff_layout_read_pagelist()
1608 	u32 idx = hdr->pgio_mirror_idx;  in ff_layout_read_pagelist()
1613 		__func__, hdr->inode->i_ino,  in ff_layout_read_pagelist()
1614 		hdr->args.pgbase, (size_t)hdr->args.count, offset);  in ff_layout_read_pagelist()
1621 						   hdr->inode);  in ff_layout_read_pagelist()
1625 	ds_cred = ff_layout_get_ds_cred(lseg, idx, hdr->cred);  in ff_layout_read_pagelist()
1635 	hdr->ds_clp = ds->ds_clp;  in ff_layout_read_pagelist()
1638 		hdr->args.fh = fh;  in ff_layout_read_pagelist()
1643 	hdr->args.offset = offset;  in ff_layout_read_pagelist()
1644 	hdr->mds_offset = offset;  in ff_layout_read_pagelist()
1647 	nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,  in ff_layout_read_pagelist()
1662 ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)  in ff_layout_write_pagelist()  argument
1664 	struct pnfs_layout_segment *lseg = hdr->lseg;  in ff_layout_write_pagelist()
1668 	loff_t offset = hdr->args.offset;  in ff_layout_write_pagelist()
1671 	int idx = hdr->pgio_mirror_idx;  in ff_layout_write_pagelist()
1678 						   hdr->inode);  in ff_layout_write_pagelist()
1682 	ds_cred = ff_layout_get_ds_cred(lseg, idx, hdr->cred);  in ff_layout_write_pagelist()
1689 		__func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count,  in ff_layout_write_pagelist()
1693 	hdr->pgio_done_cb = ff_layout_write_done_cb;  in ff_layout_write_pagelist()
1695 	hdr->ds_clp = ds->ds_clp;  in ff_layout_write_pagelist()
1696 	hdr->ds_commit_idx = idx;  in ff_layout_write_pagelist()
1699 		hdr->args.fh = fh;  in ff_layout_write_pagelist()
1705 	hdr->args.offset = offset;  in ff_layout_write_pagelist()
1708 	nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,  in ff_layout_write_pagelist()
1808 	struct pnfs_layout_hdr *hdr = &flo->generic_hdr;  in ff_layout_encode_ioerr()  local
1817 	spin_lock(&hdr->plh_inode->i_lock);  in ff_layout_encode_ioerr()
1819 	spin_unlock(&hdr->plh_inode->i_lock);  in ff_layout_encode_ioerr()