107 static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
110 static inline void get_dreq(struct nfs_direct_req *dreq)  in get_dreq()  argument
112 	atomic_inc(&dreq->io_count);  in get_dreq()
115 static inline int put_dreq(struct nfs_direct_req *dreq)  in put_dreq()  argument
117 	return atomic_dec_and_test(&dreq->io_count);  in put_dreq()
120 void nfs_direct_set_resched_writes(struct nfs_direct_req *dreq)  in nfs_direct_set_resched_writes()  argument
122 	dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_set_resched_writes()
127 nfs_direct_good_bytes(struct nfs_direct_req *dreq, struct nfs_pgio_header *hdr)  in nfs_direct_good_bytes()  argument
132 	if (dreq->mirror_count == 1) {  in nfs_direct_good_bytes()
133 		dreq->mirrors[hdr->pgio_mirror_idx].count += hdr->good_bytes;  in nfs_direct_good_bytes()
134 		dreq->count += hdr->good_bytes;  in nfs_direct_good_bytes()
137 		count = dreq->mirrors[hdr->pgio_mirror_idx].count;  in nfs_direct_good_bytes()
138 		if (count + dreq->io_start < hdr->io_start + hdr->good_bytes) {  in nfs_direct_good_bytes()
139 			count = hdr->io_start + hdr->good_bytes - dreq->io_start;  in nfs_direct_good_bytes()
140 			dreq->mirrors[hdr->pgio_mirror_idx].count = count;  in nfs_direct_good_bytes()
144 		count = dreq->mirrors[0].count;  in nfs_direct_good_bytes()
146 		for (i = 1; i < dreq->mirror_count; i++)  in nfs_direct_good_bytes()
147 			count = min(count, dreq->mirrors[i].count);  in nfs_direct_good_bytes()
149 		dreq->count = count;  in nfs_direct_good_bytes()
162 nfs_direct_select_verf(struct nfs_direct_req *dreq,  in nfs_direct_select_verf()  argument
166 	struct nfs_writeverf *verfp = &dreq->verf;  in nfs_direct_select_verf()
173 	if (ds_clp && dreq->ds_cinfo.nbuckets > 0) {  in nfs_direct_select_verf()
174 		if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets)  in nfs_direct_select_verf()
175 			verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf;  in nfs_direct_select_verf()
191 static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq,  in nfs_direct_set_hdr_verf()  argument
196 	verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);  in nfs_direct_set_hdr_verf()
211 static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq,  in nfs_direct_set_or_cmp_hdr_verf()  argument
216 	verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);  in nfs_direct_set_or_cmp_hdr_verf()
218 		nfs_direct_set_hdr_verf(dreq, hdr);  in nfs_direct_set_or_cmp_hdr_verf()
232 static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq,  in nfs_direct_cmp_commit_data_verf()  argument
237 	verfp = nfs_direct_select_verf(dreq, data->ds_clp,  in nfs_direct_cmp_commit_data_verf()
282 			      struct nfs_direct_req *dreq)  in nfs_init_cinfo_from_dreq()  argument
284 	cinfo->lock = &dreq->inode->i_lock;  in nfs_init_cinfo_from_dreq()
285 	cinfo->mds = &dreq->mds_cinfo;  in nfs_init_cinfo_from_dreq()
286 	cinfo->ds = &dreq->ds_cinfo;  in nfs_init_cinfo_from_dreq()
287 	cinfo->dreq = dreq;  in nfs_init_cinfo_from_dreq()
291 static inline void nfs_direct_setup_mirroring(struct nfs_direct_req *dreq,  in nfs_direct_setup_mirroring()  argument
300 	dreq->mirror_count = mirror_count;  in nfs_direct_setup_mirroring()
305 	struct nfs_direct_req *dreq;  in nfs_direct_req_alloc()  local
307 	dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);  in nfs_direct_req_alloc()
308 	if (!dreq)  in nfs_direct_req_alloc()
311 	kref_init(&dreq->kref);  in nfs_direct_req_alloc()
312 	kref_get(&dreq->kref);  in nfs_direct_req_alloc()
313 	init_completion(&dreq->completion);  in nfs_direct_req_alloc()
314 	INIT_LIST_HEAD(&dreq->mds_cinfo.list);  in nfs_direct_req_alloc()
315 	dreq->verf.committed = NFS_INVALID_STABLE_HOW;	/* not set yet */  in nfs_direct_req_alloc()
316 	INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);  in nfs_direct_req_alloc()
317 	dreq->mirror_count = 1;  in nfs_direct_req_alloc()
318 	spin_lock_init(&dreq->lock);  in nfs_direct_req_alloc()
320 	return dreq;  in nfs_direct_req_alloc()
325 	struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);  in nfs_direct_req_free()  local
327 	nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo);  in nfs_direct_req_free()
328 	if (dreq->l_ctx != NULL)  in nfs_direct_req_free()
329 		nfs_put_lock_context(dreq->l_ctx);  in nfs_direct_req_free()
330 	if (dreq->ctx != NULL)  in nfs_direct_req_free()
331 		put_nfs_open_context(dreq->ctx);  in nfs_direct_req_free()
332 	kmem_cache_free(nfs_direct_cachep, dreq);  in nfs_direct_req_free()
335 static void nfs_direct_req_release(struct nfs_direct_req *dreq)  in nfs_direct_req_release()  argument
337 	kref_put(&dreq->kref, nfs_direct_req_free);  in nfs_direct_req_release()
340 ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)  in nfs_dreq_bytes_left()  argument
342 	return dreq->bytes_left;  in nfs_dreq_bytes_left()
349 static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)  in nfs_direct_wait()  argument
354 	if (dreq->iocb)  in nfs_direct_wait()
357 	result = wait_for_completion_killable(&dreq->completion);  in nfs_direct_wait()
360 		result = dreq->error;  in nfs_direct_wait()
362 		result = dreq->count;  in nfs_direct_wait()
372 static void nfs_direct_complete(struct nfs_direct_req *dreq, bool write)  in nfs_direct_complete()  argument
374 	struct inode *inode = dreq->inode;  in nfs_direct_complete()
376 	if (dreq->iocb && write) {  in nfs_direct_complete()
377 		loff_t pos = dreq->iocb->ki_pos + dreq->count;  in nfs_direct_complete()
390 	if (dreq->iocb) {  in nfs_direct_complete()
391 		long res = (long) dreq->error;  in nfs_direct_complete()
393 			res = (long) dreq->count;  in nfs_direct_complete()
394 		dreq->iocb->ki_complete(dreq->iocb, res, 0);  in nfs_direct_complete()
397 	complete_all(&dreq->completion);  in nfs_direct_complete()
399 	nfs_direct_req_release(dreq);  in nfs_direct_complete()
415 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_read_completion()  local
420 	spin_lock(&dreq->lock);  in nfs_direct_read_completion()
422 		dreq->error = hdr->error;  in nfs_direct_read_completion()
424 		nfs_direct_good_bytes(dreq, hdr);  in nfs_direct_read_completion()
426 	spin_unlock(&dreq->lock);  in nfs_direct_read_completion()
439 	if (put_dreq(dreq))  in nfs_direct_read_completion()
440 		nfs_direct_complete(dreq, false);  in nfs_direct_read_completion()
457 	get_dreq(hdr->dreq);  in nfs_direct_pgio_init()
474 static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_read_schedule_iovec()  argument
479 	struct inode *inode = dreq->inode;  in nfs_direct_read_schedule_iovec()
484 	nfs_pageio_init_read(&desc, dreq->inode, false,  in nfs_direct_read_schedule_iovec()
486 	get_dreq(dreq);  in nfs_direct_read_schedule_iovec()
487 	desc.pg_dreq = dreq;  in nfs_direct_read_schedule_iovec()
508 			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,  in nfs_direct_read_schedule_iovec()
525 			dreq->bytes_left -= req_len;  in nfs_direct_read_schedule_iovec()
541 		nfs_direct_req_release(dreq);  in nfs_direct_read_schedule_iovec()
545 	if (put_dreq(dreq))  in nfs_direct_read_schedule_iovec()
546 		nfs_direct_complete(dreq, false);  in nfs_direct_read_schedule_iovec()
576 	struct nfs_direct_req *dreq;  in nfs_file_direct_read()  local
597 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_read()
598 	if (dreq == NULL)  in nfs_file_direct_read()
601 	dreq->inode = inode;  in nfs_file_direct_read()
602 	dreq->bytes_left = count;  in nfs_file_direct_read()
603 	dreq->io_start = pos;  in nfs_file_direct_read()
604 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_read()
605 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_read()
610 	dreq->l_ctx = l_ctx;  in nfs_file_direct_read()
612 		dreq->iocb = iocb;  in nfs_file_direct_read()
615 	result = nfs_direct_read_schedule_iovec(dreq, iter, pos);  in nfs_file_direct_read()
620 		result = nfs_direct_wait(dreq);  in nfs_file_direct_read()
625 	nfs_direct_req_release(dreq);  in nfs_file_direct_read()
629 	nfs_direct_req_release(dreq);  in nfs_file_direct_read()
650 static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)  in nfs_direct_write_reschedule()  argument
659 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_reschedule()
660 	nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);  in nfs_direct_write_reschedule()
662 	dreq->count = 0;  in nfs_direct_write_reschedule()
663 	for (i = 0; i < dreq->mirror_count; i++)  in nfs_direct_write_reschedule()
664 		dreq->mirrors[i].count = 0;  in nfs_direct_write_reschedule()
665 	get_dreq(dreq);  in nfs_direct_write_reschedule()
667 	nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,  in nfs_direct_write_reschedule()
669 	desc.pg_dreq = dreq;  in nfs_direct_write_reschedule()
672 	nfs_direct_setup_mirroring(dreq, &desc, req);  in nfs_direct_write_reschedule()
679 			dreq->flags = 0;  in nfs_direct_write_reschedule()
680 			dreq->error = -EIO;  in nfs_direct_write_reschedule()
693 	if (put_dreq(dreq))  in nfs_direct_write_reschedule()
694 		nfs_direct_write_complete(dreq, dreq->inode);  in nfs_direct_write_reschedule()
699 	struct nfs_direct_req *dreq = data->dreq;  in nfs_direct_commit_complete()  local
704 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_complete()
708 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_commit_complete()
709 	} else if (nfs_direct_cmp_commit_data_verf(dreq, data)) {  in nfs_direct_commit_complete()
711 		dreq->flags = NFS_ODIRECT_RESCHED_WRITES;  in nfs_direct_commit_complete()
718 		if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {  in nfs_direct_commit_complete()
727 		nfs_direct_write_complete(dreq, data->inode);  in nfs_direct_commit_complete()
740 static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)  in nfs_direct_commit_schedule()  argument
746 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_commit_schedule()
747 	nfs_scan_commit(dreq->inode, &mds_list, &cinfo);  in nfs_direct_commit_schedule()
748 	res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);  in nfs_direct_commit_schedule()
750 		nfs_direct_write_reschedule(dreq);  in nfs_direct_commit_schedule()
755 	struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);  in nfs_direct_write_schedule_work()  local
756 	int flags = dreq->flags;  in nfs_direct_write_schedule_work()
758 	dreq->flags = 0;  in nfs_direct_write_schedule_work()
761 			nfs_direct_commit_schedule(dreq);  in nfs_direct_write_schedule_work()
764 			nfs_direct_write_reschedule(dreq);  in nfs_direct_write_schedule_work()
767 			nfs_direct_complete(dreq, true);  in nfs_direct_write_schedule_work()
771 static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)  in nfs_direct_write_complete()  argument
773 	schedule_work(&dreq->work); /* Calls nfs_direct_write_schedule_work */  in nfs_direct_write_complete()
778 	struct nfs_direct_req *dreq = hdr->dreq;  in nfs_direct_write_completion()  local
786 	nfs_init_cinfo_from_dreq(&cinfo, dreq);  in nfs_direct_write_completion()
788 	spin_lock(&dreq->lock);  in nfs_direct_write_completion()
791 		dreq->flags = 0;  in nfs_direct_write_completion()
792 		dreq->error = hdr->error;  in nfs_direct_write_completion()
794 	if (dreq->error == 0) {  in nfs_direct_write_completion()
795 		nfs_direct_good_bytes(dreq, hdr);  in nfs_direct_write_completion()
797 			if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)  in nfs_direct_write_completion()
799 			else if (dreq->flags == 0) {  in nfs_direct_write_completion()
800 				nfs_direct_set_hdr_verf(dreq, hdr);  in nfs_direct_write_completion()
802 				dreq->flags = NFS_ODIRECT_DO_COMMIT;  in nfs_direct_write_completion()
803 			} else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {  in nfs_direct_write_completion()
805 				if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr))  in nfs_direct_write_completion()
806 					dreq->flags =  in nfs_direct_write_completion()
811 	spin_unlock(&dreq->lock);  in nfs_direct_write_completion()
826 	if (put_dreq(dreq))  in nfs_direct_write_completion()
827 		nfs_direct_write_complete(dreq, hdr->inode);  in nfs_direct_write_completion()
860 static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,  in nfs_direct_write_schedule_iovec()  argument
865 	struct inode *inode = dreq->inode;  in nfs_direct_write_schedule_iovec()
872 	desc.pg_dreq = dreq;  in nfs_direct_write_schedule_iovec()
873 	get_dreq(dreq);  in nfs_direct_write_schedule_iovec()
895 			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,  in nfs_direct_write_schedule_iovec()
902 			nfs_direct_setup_mirroring(dreq, &desc, req);  in nfs_direct_write_schedule_iovec()
916 			dreq->bytes_left -= req_len;  in nfs_direct_write_schedule_iovec()
931 		nfs_direct_req_release(dreq);  in nfs_direct_write_schedule_iovec()
935 	if (put_dreq(dreq))  in nfs_direct_write_schedule_iovec()
936 		nfs_direct_write_complete(dreq, dreq->inode);  in nfs_direct_write_schedule_iovec()
967 	struct nfs_direct_req *dreq;  in nfs_file_direct_write()  local
996 	dreq = nfs_direct_req_alloc();  in nfs_file_direct_write()
997 	if (!dreq)  in nfs_file_direct_write()
1000 	dreq->inode = inode;  in nfs_file_direct_write()
1001 	dreq->bytes_left = iov_iter_count(iter);  in nfs_file_direct_write()
1002 	dreq->io_start = pos;  in nfs_file_direct_write()
1003 	dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));  in nfs_file_direct_write()
1004 	l_ctx = nfs_get_lock_context(dreq->ctx);  in nfs_file_direct_write()
1009 	dreq->l_ctx = l_ctx;  in nfs_file_direct_write()
1011 		dreq->iocb = iocb;  in nfs_file_direct_write()
1013 	result = nfs_direct_write_schedule_iovec(dreq, iter, pos);  in nfs_file_direct_write()
1023 		result = nfs_direct_wait(dreq);  in nfs_file_direct_write()
1035 	nfs_direct_req_release(dreq);  in nfs_file_direct_write()
1039 	nfs_direct_req_release(dreq);  in nfs_file_direct_write()