58 static void rdma_build_arg_xdr(struct svc_rqst *rqstp,  in rdma_build_arg_xdr()  argument
69 	put_page(rqstp->rq_pages[0]);  in rdma_build_arg_xdr()
70 	rqstp->rq_pages[0] = page;  in rdma_build_arg_xdr()
73 	rqstp->rq_arg.head[0].iov_base = page_address(page);  in rdma_build_arg_xdr()
74 	rqstp->rq_arg.head[0].iov_len =  in rdma_build_arg_xdr()
76 	rqstp->rq_arg.len = byte_count;  in rdma_build_arg_xdr()
77 	rqstp->rq_arg.buflen = byte_count;  in rdma_build_arg_xdr()
80 	bc = byte_count - rqstp->rq_arg.head[0].iov_len;  in rdma_build_arg_xdr()
83 	rqstp->rq_arg.page_len = bc;  in rdma_build_arg_xdr()
84 	rqstp->rq_arg.page_base = 0;  in rdma_build_arg_xdr()
87 	rmsgp = (struct rpcrdma_msg *)rqstp->rq_arg.head[0].iov_base;  in rdma_build_arg_xdr()
89 		rqstp->rq_arg.pages = &rqstp->rq_pages[0];  in rdma_build_arg_xdr()
91 		rqstp->rq_arg.pages = &rqstp->rq_pages[1];  in rdma_build_arg_xdr()
96 		put_page(rqstp->rq_pages[sge_no]);  in rdma_build_arg_xdr()
97 		rqstp->rq_pages[sge_no] = page;  in rdma_build_arg_xdr()
99 		rqstp->rq_arg.buflen += ctxt->sge[sge_no].length;  in rdma_build_arg_xdr()
102 	rqstp->rq_respages = &rqstp->rq_pages[sge_no];  in rdma_build_arg_xdr()
103 	rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_build_arg_xdr()
114 	rqstp->rq_arg.tail[0].iov_base = NULL;  in rdma_build_arg_xdr()
115 	rqstp->rq_arg.tail[0].iov_len = 0;  in rdma_build_arg_xdr()
120 			struct svc_rqst *rqstp,  in rdma_read_chunk_lcl()  argument
145 		head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no];  in rdma_read_chunk_lcl()
150 		rqstp->rq_respages = &rqstp->rq_arg.pages[pg_no+1];  in rdma_read_chunk_lcl()
151 		rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_read_chunk_lcl()
213 			 struct svc_rqst *rqstp,  in rdma_read_chunk_frmr()  argument
248 		head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no];  in rdma_read_chunk_frmr()
254 		sg_set_page(&frmr->sg[pno], rqstp->rq_arg.pages[pg_no],  in rdma_read_chunk_frmr()
257 		rqstp->rq_respages = &rqstp->rq_arg.pages[pg_no+1];  in rdma_read_chunk_frmr()
258 		rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_read_chunk_frmr()
371 rdma_copy_tail(struct svc_rqst *rqstp, struct svc_rdma_op_ctxt *head,  in rdma_copy_tail()  argument
387 		destp = page_address(rqstp->rq_arg.pages[page_no]);  in rdma_copy_tail()
401 	destp = page_address(rqstp->rq_arg.pages[page_no]);  in rdma_copy_tail()
405 	rqstp->rq_respages = &rqstp->rq_arg.pages[page_no+1];  in rdma_copy_tail()
406 	rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_copy_tail()
418 			    struct svc_rqst *rqstp,  in rdma_read_chunks()  argument
440 	head->arg.head[0] = rqstp->rq_arg.head[0];  in rdma_read_chunks()
441 	head->arg.tail[0] = rqstp->rq_arg.tail[0];  in rdma_read_chunks()
445 	head->arg.len = rqstp->rq_arg.len;  in rdma_read_chunks()
446 	head->arg.buflen = rqstp->rq_arg.buflen;  in rdma_read_chunks()
473 			ret = xprt->sc_reader(xprt, rqstp, head,  in rdma_read_chunks()
497 		ret = rdma_copy_tail(rqstp, head, position,  in rdma_read_chunks()
505 	     &rqstp->rq_pages[page_no] < rqstp->rq_respages; page_no++)  in rdma_read_chunks()
506 		rqstp->rq_pages[page_no] = NULL;  in rdma_read_chunks()
511 static int rdma_read_complete(struct svc_rqst *rqstp,  in rdma_read_complete()  argument
519 		put_page(rqstp->rq_pages[page_no]);  in rdma_read_complete()
520 		rqstp->rq_pages[page_no] = head->pages[page_no];  in rdma_read_complete()
538 	rqstp->rq_arg.pages = &rqstp->rq_pages[head->hdr_count];  in rdma_read_complete()
539 	rqstp->rq_arg.page_len = head->arg.page_len;  in rdma_read_complete()
540 	rqstp->rq_arg.page_base = head->arg.page_base;  in rdma_read_complete()
543 	rqstp->rq_respages = &rqstp->rq_pages[page_no];  in rdma_read_complete()
544 	rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_read_complete()
547 	rqstp->rq_arg.head[0] = head->arg.head[0];  in rdma_read_complete()
548 	rqstp->rq_arg.tail[0] = head->arg.tail[0];  in rdma_read_complete()
549 	rqstp->rq_arg.len = head->arg.len;  in rdma_read_complete()
550 	rqstp->rq_arg.buflen = head->arg.buflen;  in rdma_read_complete()
556 	rqstp->rq_prot = IPPROTO_MAX;  in rdma_read_complete()
557 	svc_xprt_copy_addrs(rqstp, rqstp->rq_xprt);  in rdma_read_complete()
559 	ret = rqstp->rq_arg.head[0].iov_len  in rdma_read_complete()
560 		+ rqstp->rq_arg.page_len  in rdma_read_complete()
561 		+ rqstp->rq_arg.tail[0].iov_len;  in rdma_read_complete()
564 		ret, rqstp->rq_arg.len,	rqstp->rq_arg.head[0].iov_base,  in rdma_read_complete()
565 		rqstp->rq_arg.head[0].iov_len);  in rdma_read_complete()
575 int svc_rdma_recvfrom(struct svc_rqst *rqstp)  in svc_rdma_recvfrom()  argument
577 	struct svc_xprt *xprt = rqstp->rq_xprt;  in svc_rdma_recvfrom()
585 	dprintk("svcrdma: rqstp=%p\n", rqstp);  in svc_rdma_recvfrom()
594 		return rdma_read_complete(rqstp, ctxt);  in svc_rdma_recvfrom()
618 		ctxt, rdma_xprt, rqstp, ctxt->wc_status);  in svc_rdma_recvfrom()
622 	rdma_build_arg_xdr(rqstp, ctxt, ctxt->byte_len);  in svc_rdma_recvfrom()
625 	len = svc_rdma_xdr_decode_req(&rmsgp, rqstp);  in svc_rdma_recvfrom()
626 	rqstp->rq_xprt_hlen = len;  in svc_rdma_recvfrom()
636 	ret = rdma_read_chunks(rdma_xprt, rmsgp, rqstp, ctxt);  in svc_rdma_recvfrom()
646 	ret = rqstp->rq_arg.head[0].iov_len  in svc_rdma_recvfrom()
647 		+ rqstp->rq_arg.page_len  in svc_rdma_recvfrom()
648 		+ rqstp->rq_arg.tail[0].iov_len;  in svc_rdma_recvfrom()
653 		ret, rqstp->rq_arg.len,  in svc_rdma_recvfrom()
654 		rqstp->rq_arg.head[0].iov_base,  in svc_rdma_recvfrom()
655 		rqstp->rq_arg.head[0].iov_len);  in svc_rdma_recvfrom()
656 	rqstp->rq_prot = IPPROTO_MAX;  in svc_rdma_recvfrom()
657 	svc_xprt_copy_addrs(rqstp, xprt);  in svc_rdma_recvfrom()