107 static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
110 static inline void get_dreq(struct nfs_direct_req *dreq)  in get_dreq()  argument
112 	atomic_inc(&dreq->io_count);  in get_dreq()
115 static inline int put_dreq(struct nfs_direct_req *dreq)  in put_dreq()  argument
117 	return atomic_dec_and_test(&dreq->io_count);  in put_dreq()
120 void nfs_direct_set_resched_writes(struct nfs_direct_req *dreq)  in nfs_direct_set_resched_writes()  argument
122 	dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_set_resched_writes()
127 nfs_direct_good_bytes(struct nfs_direct_req *dreq, struct nfs_pgio_header *hdr)  in nfs_direct_good_bytes()  argument
132 	if (dreq->mirror_count == 1) {  in nfs_direct_good_bytes()
133 		dreq->mirrors[hdr->pgio_mirror_idx].count += hdr->good_bytes;  in nfs_direct_good_bytes()
134 		dreq->count += hdr->good_bytes;  in nfs_direct_good_bytes()
137 		count = dreq->mirrors[hdr->pgio_mirror_idx].count;  in nfs_direct_good_bytes()
138 		if (count + dreq->io_start < hdr->io_start + hdr->good_bytes) {  in nfs_direct_good_bytes()
139 			count = hdr->io_start + hdr->good_bytes - dreq->io_start;  in nfs_direct_good_bytes()
140 			dreq->mirrors[hdr->pgio_mirror_idx].count = count;  in nfs_direct_good_bytes()
144 		count = dreq->mirrors[0].count;  in nfs_direct_good_bytes()
146 		for (i = 1; i < dreq->mirror_count; i++)  in nfs_direct_good_bytes()
147 			count = min(count, dreq->mirrors[i].count);  in nfs_direct_good_bytes()
149 		dreq->count = count;  in nfs_direct_good_bytes()
162 nfs_direct_select_verf(struct nfs_direct_req *dreq,  in nfs_direct_select_verf()  argument
166 	struct nfs_writeverf *verfp = &dreq->verf;  in nfs_direct_select_verf()
171 		if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets)  in nfs_direct_select_verf()
172 			verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf;  in nfs_direct_select_verf()
188 static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq,  in nfs_direct_set_hdr_verf()  argument
193 	verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);  in nfs_direct_set_hdr_verf()
208 static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq,  in nfs_direct_set_or_cmp_hdr_verf()  argument
213 	verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);  in nfs_direct_set_or_cmp_hdr_verf()
215 		nfs_direct_set_hdr_verf(dreq, hdr);  in nfs_direct_set_or_cmp_hdr_verf()
229 static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq,  in nfs_direct_cmp_commit_data_verf()  argument
234 	verfp = nfs_direct_select_verf(dreq, data->ds_clp,  in nfs_direct_cmp_commit_data_verf()
279 			      struct nfs_direct_req *dreq)  in nfs_init_cinfo_from_dreq()  argument
281 	cinfo->lock = &dreq->inode->i_lock;  in nfs_init_cinfo_from_dreq()
282 	cinfo->mds = &dreq->mds_cinfo;  in nfs_init_cinfo_from_dreq()
283 	cinfo->ds = &dreq->ds_cinfo;  in nfs_init_cinfo_from_dreq()
284 	cinfo->dreq = dreq;  in nfs_init_cinfo_from_dreq()
288 static inline void nfs_direct_setup_mirroring(struct nfs_direct_req *dreq,  in nfs_direct_setup_mirroring()  argument
297 	dreq->mirror_count = mirror_count;  in nfs_direct_setup_mirroring()
302 	struct nfs_direct_req *dreq;  in nfs_direct_req_alloc()  local
304 	dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);  in nfs_direct_req_alloc()
305 	if (!dreq)  in nfs_direct_req_alloc()
308 	kref_init(&dreq->kref);  in nfs_direct_req_alloc()
309 	kref_get(&dreq->kref);  in nfs_direct_req_alloc()
310 	init_completion(&dreq->completion);  in nfs_direct_req_alloc()
311 	INIT_LIST_HEAD(&dreq->mds_cinfo.list);  in nfs_direct_req_alloc()
312 	dreq->verf.committed = NFS_INVALID_STABLE_HOW;	/* not set yet */  in nfs_direct_req_alloc()
313 	INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);  in nfs_direct_req_alloc()
314 	dreq->mirror_count = 1;  in nfs_direct_req_alloc()
315 	spin_lock_init(&dreq->lock);  in nfs_direct_req_alloc()
317 	return dreq;  in nfs_direct_req_alloc()
322 	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);  in nfs_direct_req_free()  local
324 	nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo);  in nfs_direct_req_free()
325 	if (dreq->l_ctx != NULL)  in nfs_direct_req_free()
326 		nfs_put_lock_context(dreq->l_ctx);  in nfs_direct_req_free()
327 	if (dreq->ctx != NULL)  in nfs_direct_req_free()
328 		put_nfs_open_context(dreq->ctx);  in nfs_direct_req_free()
329 	kmem_cache_free(nfs_direct_cachep, dreq);  in nfs_direct_req_free()
332 static void nfs_direct_req_release(struct nfs_direct_req *dreq)  in nfs_direct_req_release()  argument
334 	kref_put(&dreq->kref, nfs_direct_req_free);  in nfs_direct_req_release()
337 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)  in nfs_dreq_bytes_left()  argument
339 	return dreq->bytes_left;  in nfs_dreq_bytes_left()
346 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)  in nfs_direct_wait()  argument
351 	if (dreq->iocb)  in nfs_direct_wait()
354 	result = wait_for_completion_killable(&dreq->completion);  in nfs_direct_wait()
357 		result = dreq->error;  in nfs_direct_wait()
359 		result = dreq->count;  in nfs_direct_wait()
369 static void nfs_direct_complete(struct nfs_direct_req *dreq, bool write)  in nfs_direct_complete()  argument
371 	struct inode *inode = dreq->inode;  in nfs_direct_complete()
373 	if (dreq->iocb && write) {  in nfs_direct_complete()
374 		loff_t pos = dreq->iocb->ki_pos + dreq->count;  in nfs_direct_complete()
387 	if (dreq->iocb) {  in nfs_direct_complete()
388 		long res = (long) dreq->error;  in nfs_direct_complete()
390 			res = (long) dreq->count;  in nfs_direct_complete()
391 		dreq->iocb->ki_complete(dreq->iocb, res, 0);  in nfs_direct_complete()
394 	complete_all(&dreq->completion);  in nfs_direct_complete()
396 	nfs_direct_req_release(dreq);  in nfs_direct_complete()
412 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_read_completion()  local
417 	spin_lock(&dreq->lock);  in nfs_direct_read_completion()
419 		dreq->error = hdr->error;  in nfs_direct_read_completion()
421 		nfs_direct_good_bytes(dreq, hdr);  in nfs_direct_read_completion()
423 	spin_unlock(&dreq->lock);  in nfs_direct_read_completion()
436 	if (put_dreq(dreq))  in nfs_direct_read_completion()
437 		nfs_direct_complete(dreq, false);  in nfs_direct_read_completion()
454 	get_dreq(hdr->dreq);  in nfs_direct_pgio_init()
471 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_read_schedule_iovec()  argument
476 	struct inode *inode = dreq->inode;  in nfs_direct_read_schedule_iovec()
481 	nfs_pageio_init_read(&desc, dreq->inode, false,  in nfs_direct_read_schedule_iovec()
483 	get_dreq(dreq);  in nfs_direct_read_schedule_iovec()
484 	desc.pg_dreq = dreq;  in nfs_direct_read_schedule_iovec()
505 			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,  in nfs_direct_read_schedule_iovec()
522 			dreq->bytes_left -= req_len;  in nfs_direct_read_schedule_iovec()
538 		nfs_direct_req_release(dreq);  in nfs_direct_read_schedule_iovec()
542 	if (put_dreq(dreq))  in nfs_direct_read_schedule_iovec()
543 		nfs_direct_complete(dreq, false);  in nfs_direct_read_schedule_iovec()
573 	struct nfs_direct_req *dreq;  in nfs_file_direct_read()  local
594 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_read()
595 	if (dreq == NULL)  in nfs_file_direct_read()
598 	dreq->inode = inode;  in nfs_file_direct_read()
599 	dreq->bytes_left = count;  in nfs_file_direct_read()
600 	dreq->io_start = pos;  in nfs_file_direct_read()
601 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_read()
602 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_read()
607 	dreq->l_ctx = l_ctx;  in nfs_file_direct_read()
609 		dreq->iocb = iocb;  in nfs_file_direct_read()
612 	result = nfs_direct_read_schedule_iovec(dreq, iter, pos);  in nfs_file_direct_read()
617 		result = nfs_direct_wait(dreq);  in nfs_file_direct_read()
622 	nfs_direct_req_release(dreq);  in nfs_file_direct_read()
626 	nfs_direct_req_release(dreq);  in nfs_file_direct_read()
647 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)  in nfs_direct_write_reschedule()  argument
656 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_reschedule()
657 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);  in nfs_direct_write_reschedule()
659 	dreq->count = 0;  in nfs_direct_write_reschedule()
660 	for (i = 0; i < dreq->mirror_count; i++)  in nfs_direct_write_reschedule()
661 		dreq->mirrors[i].count = 0;  in nfs_direct_write_reschedule()
662 	get_dreq(dreq);  in nfs_direct_write_reschedule()
664 	nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,  in nfs_direct_write_reschedule()
666 	desc.pg_dreq = dreq;  in nfs_direct_write_reschedule()
669 	nfs_direct_setup_mirroring(dreq, &desc, req);  in nfs_direct_write_reschedule()
676 			dreq->flags = 0;  in nfs_direct_write_reschedule()
677 			dreq->error = -EIO;  in nfs_direct_write_reschedule()
690 	if (put_dreq(dreq))  in nfs_direct_write_reschedule()
691 		nfs_direct_write_complete(dreq, dreq->inode);  in nfs_direct_write_reschedule()
696 	struct nfs_direct_req *dreq = data->dreq;  in nfs_direct_commit_complete()  local
701 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_complete()
705 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_commit_complete()
706 	} else if (nfs_direct_cmp_commit_data_verf(dreq, data)) {  in nfs_direct_commit_complete()
708 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_commit_complete()
715 		if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {  in nfs_direct_commit_complete()
724 		nfs_direct_write_complete(dreq, data->inode);  in nfs_direct_commit_complete()
737 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)  in nfs_direct_commit_schedule()  argument
743 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_schedule()
744 	nfs_scan_commit(dreq->inode, &mds_list, &cinfo);  in nfs_direct_commit_schedule()
745 	res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);  in nfs_direct_commit_schedule()
747 		nfs_direct_write_reschedule(dreq);  in nfs_direct_commit_schedule()
752 	struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);  in nfs_direct_write_schedule_work()  local
753 	int flags = dreq->flags;  in nfs_direct_write_schedule_work()
755 	dreq->flags = 0;  in nfs_direct_write_schedule_work()
758 			nfs_direct_commit_schedule(dreq);  in nfs_direct_write_schedule_work()
761 			nfs_direct_write_reschedule(dreq);  in nfs_direct_write_schedule_work()
764 			nfs_direct_complete(dreq, true);  in nfs_direct_write_schedule_work()
768 static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)  in nfs_direct_write_complete()  argument
770 	schedule_work(&dreq->work); /* Calls nfs_direct_write_schedule_work */  in nfs_direct_write_complete()
775 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_completion()  local
783 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_completion()
785 	spin_lock(&dreq->lock);  in nfs_direct_write_completion()
788 		dreq->flags = 0;  in nfs_direct_write_completion()
789 		dreq->error = hdr->error;  in nfs_direct_write_completion()
791 	if (dreq->error == 0) {  in nfs_direct_write_completion()
792 		nfs_direct_good_bytes(dreq, hdr);  in nfs_direct_write_completion()
794 			if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)  in nfs_direct_write_completion()
796 			else if (dreq->flags == 0) {  in nfs_direct_write_completion()
797 				nfs_direct_set_hdr_verf(dreq, hdr);  in nfs_direct_write_completion()
799 				dreq->flags = NFS_ODIRECT_DO_COMMIT;  in nfs_direct_write_completion()
800 			} else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {  in nfs_direct_write_completion()
802 				if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr))  in nfs_direct_write_completion()
803 					dreq->flags =  in nfs_direct_write_completion()
808 	spin_unlock(&dreq->lock);  in nfs_direct_write_completion()
823 	if (put_dreq(dreq))  in nfs_direct_write_completion()
824 		nfs_direct_write_complete(dreq, hdr->inode);  in nfs_direct_write_completion()
857 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_write_schedule_iovec()  argument
862 	struct inode *inode = dreq->inode;  in nfs_direct_write_schedule_iovec()
869 	desc.pg_dreq = dreq;  in nfs_direct_write_schedule_iovec()
870 	get_dreq(dreq);  in nfs_direct_write_schedule_iovec()
892 			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,  in nfs_direct_write_schedule_iovec()
899 			nfs_direct_setup_mirroring(dreq, &desc, req);  in nfs_direct_write_schedule_iovec()
913 			dreq->bytes_left -= req_len;  in nfs_direct_write_schedule_iovec()
928 		nfs_direct_req_release(dreq);  in nfs_direct_write_schedule_iovec()
932 	if (put_dreq(dreq))  in nfs_direct_write_schedule_iovec()
933 		nfs_direct_write_complete(dreq, dreq->inode);  in nfs_direct_write_schedule_iovec()
964 	struct nfs_direct_req *dreq;  in nfs_file_direct_write()  local
993 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_write()
994 	if (!dreq)  in nfs_file_direct_write()
997 	dreq->inode = inode;  in nfs_file_direct_write()
998 	dreq->bytes_left = iov_iter_count(iter);  in nfs_file_direct_write()
999 	dreq->io_start = pos;  in nfs_file_direct_write()
1000 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_write()
1001 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_write()
1006 	dreq->l_ctx = l_ctx;  in nfs_file_direct_write()
1008 		dreq->iocb = iocb;  in nfs_file_direct_write()
1010 	result = nfs_direct_write_schedule_iovec(dreq, iter, pos);  in nfs_file_direct_write()
1020 		result = nfs_direct_wait(dreq);  in nfs_file_direct_write()
1032 	nfs_direct_req_release(dreq);  in nfs_file_direct_write()
1036 	nfs_direct_req_release(dreq);  in nfs_file_direct_write()