23 static int svc_deferred_recv(struct svc_rqst *rqstp);
286 void svc_xprt_copy_addrs(struct svc_rqst *rqstp, struct svc_xprt *xprt)  in svc_xprt_copy_addrs()  argument
288 	memcpy(&rqstp->rq_addr, &xprt->xpt_remote, xprt->xpt_remotelen);  in svc_xprt_copy_addrs()
289 	rqstp->rq_addrlen = xprt->xpt_remotelen;  in svc_xprt_copy_addrs()
295 	memcpy(&rqstp->rq_daddr, &xprt->xpt_local, xprt->xpt_locallen);  in svc_xprt_copy_addrs()
296 	rqstp->rq_daddrlen = xprt->xpt_locallen;  in svc_xprt_copy_addrs()
307 char *svc_print_addr(struct svc_rqst *rqstp, char *buf, size_t len)  in svc_print_addr()  argument
309 	return __svc_print_addr(svc_addr(rqstp), buf, len);  in svc_print_addr()
325 	struct svc_rqst	*rqstp = NULL;  in svc_xprt_do_enqueue()  local
351 	list_for_each_entry_rcu(rqstp, &pool->sp_all_threads, rq_all) {  in svc_xprt_do_enqueue()
353 		if (test_bit(RQ_BUSY, &rqstp->rq_flags))  in svc_xprt_do_enqueue()
363 			spin_lock_bh(&rqstp->rq_lock);  in svc_xprt_do_enqueue()
364 			if (test_and_set_bit(RQ_BUSY, &rqstp->rq_flags)) {  in svc_xprt_do_enqueue()
366 				spin_unlock_bh(&rqstp->rq_lock);  in svc_xprt_do_enqueue()
371 			rqstp->rq_xprt = xprt;  in svc_xprt_do_enqueue()
373 			spin_unlock_bh(&rqstp->rq_lock);  in svc_xprt_do_enqueue()
378 		wake_up_process(rqstp->rq_task);  in svc_xprt_do_enqueue()
399 	rqstp = NULL;  in svc_xprt_do_enqueue()
402 	trace_svc_xprt_do_enqueue(xprt, rqstp);  in svc_xprt_do_enqueue()
455 void svc_reserve(struct svc_rqst *rqstp, int space)  in svc_reserve()  argument
457 	space += rqstp->rq_res.head[0].iov_len;  in svc_reserve()
459 	if (space < rqstp->rq_reserved) {  in svc_reserve()
460 		struct svc_xprt *xprt = rqstp->rq_xprt;  in svc_reserve()
461 		atomic_sub((rqstp->rq_reserved - space), &xprt->xpt_reserved);  in svc_reserve()
462 		rqstp->rq_reserved = space;  in svc_reserve()
471 static void svc_xprt_release(struct svc_rqst *rqstp)  in svc_xprt_release()  argument
473 	struct svc_xprt	*xprt = rqstp->rq_xprt;  in svc_xprt_release()
475 	rqstp->rq_xprt->xpt_ops->xpo_release_rqst(rqstp);  in svc_xprt_release()
477 	kfree(rqstp->rq_deferred);  in svc_xprt_release()
478 	rqstp->rq_deferred = NULL;  in svc_xprt_release()
480 	svc_free_res_pages(rqstp);  in svc_xprt_release()
481 	rqstp->rq_res.page_len = 0;  in svc_xprt_release()
482 	rqstp->rq_res.page_base = 0;  in svc_xprt_release()
489 	if ((rqstp->rq_res.len) >  rqstp->rq_reserved)  in svc_xprt_release()
491 		       rqstp->rq_reserved,  in svc_xprt_release()
492 		       rqstp->rq_res.len);  in svc_xprt_release()
494 	rqstp->rq_res.head[0].iov_len = 0;  in svc_xprt_release()
495 	svc_reserve(rqstp, 0);  in svc_xprt_release()
496 	rqstp->rq_xprt = NULL;  in svc_xprt_release()
510 	struct svc_rqst	*rqstp;  in svc_wake_up()  local
516 	list_for_each_entry_rcu(rqstp, &pool->sp_all_threads, rq_all) {  in svc_wake_up()
518 		if (test_bit(RQ_BUSY, &rqstp->rq_flags))  in svc_wake_up()
521 		dprintk("svc: daemon %p woken up.\n", rqstp);  in svc_wake_up()
522 		wake_up_process(rqstp->rq_task);  in svc_wake_up()
523 		trace_svc_wake_up(rqstp->rq_task->pid);  in svc_wake_up()
600 static int svc_alloc_arg(struct svc_rqst *rqstp)  in svc_alloc_arg()  argument
602 	struct svc_serv *serv = rqstp->rq_server;  in svc_alloc_arg()
614 		while (rqstp->rq_pages[i] == NULL) {  in svc_alloc_arg()
624 			rqstp->rq_pages[i] = p;  in svc_alloc_arg()
626 	rqstp->rq_page_end = &rqstp->rq_pages[i];  in svc_alloc_arg()
627 	rqstp->rq_pages[i++] = NULL; /* this might be seen in nfs_read_actor */  in svc_alloc_arg()
630 	arg = &rqstp->rq_arg;  in svc_alloc_arg()
631 	arg->head[0].iov_base = page_address(rqstp->rq_pages[0]);  in svc_alloc_arg()
633 	arg->pages = rqstp->rq_pages + 1;  in svc_alloc_arg()
643 rqst_should_sleep(struct svc_rqst *rqstp)  in rqst_should_sleep()  argument
645 	struct svc_pool		*pool = rqstp->rq_pool;  in rqst_should_sleep()
666 static struct svc_xprt *svc_get_next_xprt(struct svc_rqst *rqstp, long timeout)  in svc_get_next_xprt()  argument
669 	struct svc_pool		*pool = rqstp->rq_pool;  in svc_get_next_xprt()
673 	WARN_ON_ONCE(rqstp->rq_xprt);  in svc_get_next_xprt()
678 	rqstp->rq_chandle.thread_wait = 5*HZ;  in svc_get_next_xprt()
682 		rqstp->rq_xprt = xprt;  in svc_get_next_xprt()
688 		rqstp->rq_chandle.thread_wait = 1*HZ;  in svc_get_next_xprt()
698 	clear_bit(RQ_BUSY, &rqstp->rq_flags);  in svc_get_next_xprt()
701 	if (likely(rqst_should_sleep(rqstp)))  in svc_get_next_xprt()
708 	spin_lock_bh(&rqstp->rq_lock);  in svc_get_next_xprt()
709 	set_bit(RQ_BUSY, &rqstp->rq_flags);  in svc_get_next_xprt()
710 	spin_unlock_bh(&rqstp->rq_lock);  in svc_get_next_xprt()
712 	xprt = rqstp->rq_xprt;  in svc_get_next_xprt()
741 static int svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt)  in svc_handle_xprt()  argument
743 	struct svc_serv *serv = rqstp->rq_server;  in svc_handle_xprt()
768 			rqstp, rqstp->rq_pool->sp_id, xprt,  in svc_handle_xprt()
770 		rqstp->rq_deferred = svc_deferred_dequeue(xprt);  in svc_handle_xprt()
771 		if (rqstp->rq_deferred)  in svc_handle_xprt()
772 			len = svc_deferred_recv(rqstp);  in svc_handle_xprt()
774 			len = xprt->xpt_ops->xpo_recvfrom(rqstp);  in svc_handle_xprt()
776 		rqstp->rq_reserved = serv->sv_max_mesg;  in svc_handle_xprt()
777 		atomic_add(rqstp->rq_reserved, &xprt->xpt_reserved);  in svc_handle_xprt()
791 int svc_recv(struct svc_rqst *rqstp, long timeout)  in svc_recv()  argument
794 	struct svc_serv		*serv = rqstp->rq_server;  in svc_recv()
798 		rqstp, timeout);  in svc_recv()
800 	if (rqstp->rq_xprt)  in svc_recv()
803 			 rqstp);  in svc_recv()
805 	err = svc_alloc_arg(rqstp);  in svc_recv()
815 	xprt = svc_get_next_xprt(rqstp, timeout);  in svc_recv()
821 	len = svc_handle_xprt(rqstp, xprt);  in svc_recv()
830 	if (xprt->xpt_ops->xpo_secure_port(rqstp))  in svc_recv()
831 		set_bit(RQ_SECURE, &rqstp->rq_flags);  in svc_recv()
833 		clear_bit(RQ_SECURE, &rqstp->rq_flags);  in svc_recv()
834 	rqstp->rq_chandle.defer = svc_defer;  in svc_recv()
835 	rqstp->rq_xid = svc_getu32(&rqstp->rq_arg.head[0]);  in svc_recv()
839 	trace_svc_recv(rqstp, len);  in svc_recv()
842 	rqstp->rq_res.len = 0;  in svc_recv()
843 	svc_xprt_release(rqstp);  in svc_recv()
845 	trace_svc_recv(rqstp, err);  in svc_recv()
853 void svc_drop(struct svc_rqst *rqstp)  in svc_drop()  argument
855 	dprintk("svc: xprt %p dropped request\n", rqstp->rq_xprt);  in svc_drop()
856 	svc_xprt_release(rqstp);  in svc_drop()
863 int svc_send(struct svc_rqst *rqstp)  in svc_send()  argument
869 	xprt = rqstp->rq_xprt;  in svc_send()
874 	rqstp->rq_xprt->xpt_ops->xpo_release_rqst(rqstp);  in svc_send()
877 	xb = &rqstp->rq_res;  in svc_send()
888 		len = xprt->xpt_ops->xpo_sendto(rqstp);  in svc_send()
891 	svc_xprt_release(rqstp);  in svc_send()
896 	trace_svc_send(rqstp, len);  in svc_send()
1111 	struct svc_rqst *rqstp = container_of(req, struct svc_rqst, rq_chandle);  in svc_defer()  local
1114 	if (rqstp->rq_arg.page_len || !test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags))  in svc_defer()
1116 	if (rqstp->rq_deferred) {  in svc_defer()
1117 		dr = rqstp->rq_deferred;  in svc_defer()
1118 		rqstp->rq_deferred = NULL;  in svc_defer()
1123 		size = sizeof(struct svc_deferred_req) + rqstp->rq_arg.len;  in svc_defer()
1128 		dr->handle.owner = rqstp->rq_server;  in svc_defer()
1129 		dr->prot = rqstp->rq_prot;  in svc_defer()
1130 		memcpy(&dr->addr, &rqstp->rq_addr, rqstp->rq_addrlen);  in svc_defer()
1131 		dr->addrlen = rqstp->rq_addrlen;  in svc_defer()
1132 		dr->daddr = rqstp->rq_daddr;  in svc_defer()
1133 		dr->argslen = rqstp->rq_arg.len >> 2;  in svc_defer()
1134 		dr->xprt_hlen = rqstp->rq_xprt_hlen;  in svc_defer()
1137 		skip = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len;  in svc_defer()
1138 		memcpy(dr->args, rqstp->rq_arg.head[0].iov_base - skip,  in svc_defer()
1141 	svc_xprt_get(rqstp->rq_xprt);  in svc_defer()
1142 	dr->xprt = rqstp->rq_xprt;  in svc_defer()
1143 	set_bit(RQ_DROPME, &rqstp->rq_flags);  in svc_defer()
1152 static int svc_deferred_recv(struct svc_rqst *rqstp)  in svc_deferred_recv()  argument
1154 	struct svc_deferred_req *dr = rqstp->rq_deferred;  in svc_deferred_recv()
1157 	rqstp->rq_arg.head[0].iov_base = dr->args + (dr->xprt_hlen>>2);  in svc_deferred_recv()
1159 	rqstp->rq_arg.head[0].iov_len = (dr->argslen<<2) - dr->xprt_hlen;  in svc_deferred_recv()
1160 	rqstp->rq_arg.page_len = 0;  in svc_deferred_recv()
1162 	rqstp->rq_arg.len     = dr->argslen<<2;  in svc_deferred_recv()
1163 	rqstp->rq_prot        = dr->prot;  in svc_deferred_recv()
1164 	memcpy(&rqstp->rq_addr, &dr->addr, dr->addrlen);  in svc_deferred_recv()
1165 	rqstp->rq_addrlen     = dr->addrlen;  in svc_deferred_recv()
1167 	rqstp->rq_xprt_hlen   = dr->xprt_hlen;  in svc_deferred_recv()
1168 	rqstp->rq_daddr       = dr->daddr;  in svc_deferred_recv()
1169 	rqstp->rq_respages    = rqstp->rq_pages;  in svc_deferred_recv()