69 static void	xprt_connect_status(struct rpc_task *task);
182 int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_reserve_xprt()  argument
184 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_reserve_xprt()
188 		if (task == xprt->snd_task)  in xprt_reserve_xprt()
192 	xprt->snd_task = task;  in xprt_reserve_xprt()
200 			task->tk_pid, xprt);  in xprt_reserve_xprt()
201 	task->tk_timeout = 0;  in xprt_reserve_xprt()
202 	task->tk_status = -EAGAIN;  in xprt_reserve_xprt()
209 	rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);  in xprt_reserve_xprt()
233 int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_reserve_xprt_cong()  argument
235 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_reserve_xprt_cong()
239 		if (task == xprt->snd_task)  in xprt_reserve_xprt_cong()
244 		xprt->snd_task = task;  in xprt_reserve_xprt_cong()
247 	if (__xprt_get_cong(xprt, task)) {  in xprt_reserve_xprt_cong()
248 		xprt->snd_task = task;  in xprt_reserve_xprt_cong()
256 	dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);  in xprt_reserve_xprt_cong()
257 	task->tk_timeout = 0;  in xprt_reserve_xprt_cong()
258 	task->tk_status = -EAGAIN;  in xprt_reserve_xprt_cong()
265 	rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);  in xprt_reserve_xprt_cong()
270 static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_lock_write()  argument
275 	retval = xprt->ops->reserve_xprt(xprt, task);  in xprt_lock_write()
280 static bool __xprt_lock_write_func(struct rpc_task *task, void *data)  in __xprt_lock_write_func()  argument
285 	req = task->tk_rqstp;  in __xprt_lock_write_func()
286 	xprt->snd_task = task;  in __xprt_lock_write_func()
302 static bool __xprt_lock_write_cong_func(struct rpc_task *task, void *data)  in __xprt_lock_write_cong_func()  argument
307 	req = task->tk_rqstp;  in __xprt_lock_write_cong_func()
309 		xprt->snd_task = task;  in __xprt_lock_write_cong_func()
312 	if (__xprt_get_cong(xprt, task)) {  in __xprt_lock_write_cong_func()
313 		xprt->snd_task = task;  in __xprt_lock_write_cong_func()
332 static void xprt_task_clear_bytes_sent(struct rpc_task *task)  in xprt_task_clear_bytes_sent()  argument
334 	if (task != NULL) {  in xprt_task_clear_bytes_sent()
335 		struct rpc_rqst *req = task->tk_rqstp;  in xprt_task_clear_bytes_sent()
348 void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_xprt()  argument
350 	if (xprt->snd_task == task) {  in xprt_release_xprt()
351 		xprt_task_clear_bytes_sent(task);  in xprt_release_xprt()
366 void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_xprt_cong()  argument
368 	if (xprt->snd_task == task) {  in xprt_release_xprt_cong()
369 		xprt_task_clear_bytes_sent(task);  in xprt_release_xprt_cong()
376 static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_release_write()  argument
379 	xprt->ops->release_xprt(xprt, task);  in xprt_release_write()
388 __xprt_get_cong(struct rpc_xprt *xprt, struct rpc_task *task)  in __xprt_get_cong()  argument
390 	struct rpc_rqst *req = task->tk_rqstp;  in __xprt_get_cong()
395 			task->tk_pid, xprt->cong, xprt->cwnd);  in __xprt_get_cong()
423 void xprt_release_rqst_cong(struct rpc_task *task)  in xprt_release_rqst_cong()  argument
425 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_release_rqst_cong()
447 void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)  in xprt_adjust_cwnd()  argument
449 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_adjust_cwnd()
495 void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action)  in xprt_wait_for_buffer_space()  argument
497 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_wait_for_buffer_space()
500 	task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0;  in xprt_wait_for_buffer_space()
501 	rpc_sleep_on(&xprt->pending, task, action);  in xprt_wait_for_buffer_space()
531 void xprt_set_retrans_timeout_def(struct rpc_task *task)  in xprt_set_retrans_timeout_def()  argument
533 	task->tk_timeout = task->tk_rqstp->rq_timeout;  in xprt_set_retrans_timeout_def()
543 void xprt_set_retrans_timeout_rtt(struct rpc_task *task)  in xprt_set_retrans_timeout_rtt()  argument
545 	int timer = task->tk_msg.rpc_proc->p_timer;  in xprt_set_retrans_timeout_rtt()
546 	struct rpc_clnt *clnt = task->tk_client;  in xprt_set_retrans_timeout_rtt()
548 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_set_retrans_timeout_rtt()
551 	task->tk_timeout = rpc_calc_rto(rtt, timer);  in xprt_set_retrans_timeout_rtt()
552 	task->tk_timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries;  in xprt_set_retrans_timeout_rtt()
553 	if (task->tk_timeout > max_timeout || task->tk_timeout == 0)  in xprt_set_retrans_timeout_rtt()
554 		task->tk_timeout = max_timeout;  in xprt_set_retrans_timeout_rtt()
698 		struct rpc_task *task,  in xprt_lock_connect()  argument
706 	if (xprt->snd_task != task)  in xprt_lock_connect()
708 	xprt_task_clear_bytes_sent(task);  in xprt_lock_connect()
735 void xprt_connect(struct rpc_task *task)  in xprt_connect()  argument
737 	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;  in xprt_connect()
739 	dprintk("RPC: %5u xprt_connect xprt %p %s connected\n", task->tk_pid,  in xprt_connect()
743 		task->tk_status = -EAGAIN;  in xprt_connect()
746 	if (!xprt_lock_write(xprt, task))  in xprt_connect()
753 		task->tk_rqstp->rq_bytes_sent = 0;  in xprt_connect()
754 		task->tk_timeout = task->tk_rqstp->rq_timeout;  in xprt_connect()
755 		rpc_sleep_on(&xprt->pending, task, xprt_connect_status);  in xprt_connect()
762 		xprt->ops->connect(xprt, task);  in xprt_connect()
764 	xprt_release_write(xprt, task);  in xprt_connect()
767 static void xprt_connect_status(struct rpc_task *task)  in xprt_connect_status()  argument
769 	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;  in xprt_connect_status()
771 	if (task->tk_status == 0) {  in xprt_connect_status()
775 				task->tk_pid);  in xprt_connect_status()
779 	switch (task->tk_status) {  in xprt_connect_status()
787 		dprintk("RPC: %5u xprt_connect_status: retrying\n", task->tk_pid);  in xprt_connect_status()
791 				"out\n", task->tk_pid);  in xprt_connect_status()
795 				"server %s\n", task->tk_pid, -task->tk_status,  in xprt_connect_status()
797 		task->tk_status = -EIO;  in xprt_connect_status()
825 static void xprt_update_rtt(struct rpc_task *task)  in xprt_update_rtt()  argument
827 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_update_rtt()
828 	struct rpc_rtt *rtt = task->tk_client->cl_rtt;  in xprt_update_rtt()
829 	unsigned int timer = task->tk_msg.rpc_proc->p_timer;  in xprt_update_rtt()
846 void xprt_complete_rqst(struct rpc_task *task, int copied)  in xprt_complete_rqst()  argument
848 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_complete_rqst()
852 			task->tk_pid, ntohl(req->rq_xid), copied);  in xprt_complete_rqst()
858 		xprt_update_rtt(task);  in xprt_complete_rqst()
866 	rpc_wake_up_queued_task(&xprt->pending, task);  in xprt_complete_rqst()
870 static void xprt_timer(struct rpc_task *task)  in xprt_timer()  argument
872 	struct rpc_rqst *req = task->tk_rqstp;  in xprt_timer()
875 	if (task->tk_status != -ETIMEDOUT)  in xprt_timer()
877 	dprintk("RPC: %5u xprt_timer\n", task->tk_pid);  in xprt_timer()
882 			xprt->ops->timer(xprt, task);  in xprt_timer()
884 		task->tk_status = 0;  in xprt_timer()
898 bool xprt_prepare_transmit(struct rpc_task *task)  in xprt_prepare_transmit()  argument
900 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_prepare_transmit()
904 	dprintk("RPC: %5u xprt_prepare_transmit\n", task->tk_pid);  in xprt_prepare_transmit()
909 			task->tk_status = req->rq_reply_bytes_recvd;  in xprt_prepare_transmit()
912 		if ((task->tk_flags & RPC_TASK_NO_RETRANS_TIMEOUT)  in xprt_prepare_transmit()
915 			xprt->ops->set_retrans_timeout(task);  in xprt_prepare_transmit()
916 			rpc_sleep_on(&xprt->pending, task, xprt_timer);  in xprt_prepare_transmit()
920 	if (!xprt->ops->reserve_xprt(xprt, task)) {  in xprt_prepare_transmit()
921 		task->tk_status = -EAGAIN;  in xprt_prepare_transmit()
930 void xprt_end_transmit(struct rpc_task *task)  in xprt_end_transmit()  argument
932 	xprt_release_write(task->tk_rqstp->rq_xprt, task);  in xprt_end_transmit()
941 void xprt_transmit(struct rpc_task *task)  in xprt_transmit()  argument
943 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_transmit()
947 	dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);  in xprt_transmit()
950 		if (list_empty(&req->rq_list) && rpc_reply_expected(task)) {  in xprt_transmit()
969 	status = xprt->ops->send_request(task);  in xprt_transmit()
972 		task->tk_status = status;  in xprt_transmit()
977 	dprintk("RPC: %5u xmit complete\n", task->tk_pid);  in xprt_transmit()
978 	task->tk_flags |= RPC_TASK_SENT;  in xprt_transmit()
981 	xprt->ops->set_retrans_timeout(task);  in xprt_transmit()
994 		task->tk_status = -ENOTCONN;  in xprt_transmit()
1000 		if (!req->rq_reply_bytes_recvd && rpc_reply_expected(task))  in xprt_transmit()
1001 			rpc_sleep_on(&xprt->pending, task, xprt_timer);  in xprt_transmit()
1007 static void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_add_backlog()  argument
1010 	rpc_sleep_on(&xprt->backlog, task, NULL);  in xprt_add_backlog()
1019 static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_throttle_congested()  argument
1027 		rpc_sleep_on(&xprt->backlog, task, NULL);  in xprt_throttle_congested()
1059 void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_alloc_slot()  argument
1076 		task->tk_status = -ENOMEM;  in xprt_alloc_slot()
1079 		xprt_add_backlog(xprt, task);  in xprt_alloc_slot()
1082 		task->tk_status = -EAGAIN;  in xprt_alloc_slot()
1087 	task->tk_status = 0;  in xprt_alloc_slot()
1088 	task->tk_rqstp = req;  in xprt_alloc_slot()
1089 	xprt_request_init(task, xprt);  in xprt_alloc_slot()
1094 void xprt_lock_and_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)  in xprt_lock_and_alloc_slot()  argument
1101 	if (xprt_lock_write(xprt, task)) {  in xprt_lock_and_alloc_slot()
1102 		xprt_alloc_slot(xprt, task);  in xprt_lock_and_alloc_slot()
1103 		xprt_release_write(xprt, task);  in xprt_lock_and_alloc_slot()
1181 void xprt_reserve(struct rpc_task *task)  in xprt_reserve()  argument
1185 	task->tk_status = 0;  in xprt_reserve()
1186 	if (task->tk_rqstp != NULL)  in xprt_reserve()
1189 	task->tk_timeout = 0;  in xprt_reserve()
1190 	task->tk_status = -EAGAIN;  in xprt_reserve()
1192 	xprt = rcu_dereference(task->tk_client->cl_xprt);  in xprt_reserve()
1193 	if (!xprt_throttle_congested(xprt, task))  in xprt_reserve()
1194 		xprt->ops->alloc_slot(xprt, task);  in xprt_reserve()
1207 void xprt_retry_reserve(struct rpc_task *task)  in xprt_retry_reserve()  argument
1211 	task->tk_status = 0;  in xprt_retry_reserve()
1212 	if (task->tk_rqstp != NULL)  in xprt_retry_reserve()
1215 	task->tk_timeout = 0;  in xprt_retry_reserve()
1216 	task->tk_status = -EAGAIN;  in xprt_retry_reserve()
1218 	xprt = rcu_dereference(task->tk_client->cl_xprt);  in xprt_retry_reserve()
1219 	xprt->ops->alloc_slot(xprt, task);  in xprt_retry_reserve()
1233 static void xprt_request_init(struct rpc_task *task, struct rpc_xprt *xprt)  in xprt_request_init()  argument
1235 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_request_init()
1238 	req->rq_timeout = task->tk_client->cl_timeout->to_initval;  in xprt_request_init()
1239 	req->rq_task	= task;  in xprt_request_init()
1251 	dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid,  in xprt_request_init()
1260 void xprt_release(struct rpc_task *task)  in xprt_release()  argument
1263 	struct rpc_rqst	*req = task->tk_rqstp;  in xprt_release()
1266 		if (task->tk_client) {  in xprt_release()
1268 			xprt = rcu_dereference(task->tk_client->cl_xprt);  in xprt_release()
1269 			if (xprt->snd_task == task)  in xprt_release()
1270 				xprt_release_write(xprt, task);  in xprt_release()
1277 	if (task->tk_ops->rpc_count_stats != NULL)  in xprt_release()
1278 		task->tk_ops->rpc_count_stats(task, task->tk_calldata);  in xprt_release()
1279 	else if (task->tk_client)  in xprt_release()
1280 		rpc_count_iostats(task, task->tk_client->cl_metrics);  in xprt_release()
1282 	xprt->ops->release_xprt(xprt, task);  in xprt_release()
1284 		xprt->ops->release_request(task);  in xprt_release()
1297 	task->tk_rqstp = NULL;  in xprt_release()
1301 	dprintk("RPC: %5u release request %p\n", task->tk_pid, req);  in xprt_release()