219 static inline struct rpc_xprt *xprt_from_sock(struct sock *sk)  in xprt_from_sock()  argument
221 	return (struct rpc_xprt *) sk->sk_user_data;  in xprt_from_sock()
456 	struct sock *sk = transport->inet;  in xs_nospace()  local
469 		sk->sk_write_pending++;  in xs_nospace()
477 	sk->sk_write_space(sk);  in xs_nospace()
742 static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk)  in xs_save_old_callbacks()  argument
744 	transport->old_data_ready = sk->sk_data_ready;  in xs_save_old_callbacks()
745 	transport->old_state_change = sk->sk_state_change;  in xs_save_old_callbacks()
746 	transport->old_write_space = sk->sk_write_space;  in xs_save_old_callbacks()
747 	transport->old_error_report = sk->sk_error_report;  in xs_save_old_callbacks()
750 static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk)  in xs_restore_old_callbacks()  argument
752 	sk->sk_data_ready = transport->old_data_ready;  in xs_restore_old_callbacks()
753 	sk->sk_state_change = transport->old_state_change;  in xs_restore_old_callbacks()
754 	sk->sk_write_space = transport->old_write_space;  in xs_restore_old_callbacks()
755 	sk->sk_error_report = transport->old_error_report;  in xs_restore_old_callbacks()
780 static void xs_error_report(struct sock *sk)  in xs_error_report()  argument
785 	read_lock_bh(&sk->sk_callback_lock);  in xs_error_report()
786 	if (!(xprt = xprt_from_sock(sk)))  in xs_error_report()
789 	err = -sk->sk_err;  in xs_error_report()
793 	if (sk->sk_state == TCP_CLOSE)  in xs_error_report()
797 	trace_rpc_socket_error(xprt, sk->sk_socket, err);  in xs_error_report()
800 	read_unlock_bh(&sk->sk_callback_lock);  in xs_error_report()
806 	struct sock *sk = transport->inet;  in xs_reset_transport()  local
809 	if (sk == NULL)  in xs_reset_transport()
813 		sk_clear_memalloc(sk);  in xs_reset_transport()
818 	write_lock_bh(&sk->sk_callback_lock);  in xs_reset_transport()
822 	sk->sk_user_data = NULL;  in xs_reset_transport()
824 	xs_restore_old_callbacks(transport, sk);  in xs_reset_transport()
826 	write_unlock_bh(&sk->sk_callback_lock);  in xs_reset_transport()
911 		struct sock *sk,  in xs_local_data_read_skb()  argument
956 	struct sock *sk;  in xs_local_data_receive()  local
960 	sk = transport->inet;  in xs_local_data_receive()
961 	if (sk == NULL)  in xs_local_data_receive()
964 		skb = skb_recv_datagram(sk, 0, 1, &err);  in xs_local_data_receive()
967 		xs_local_data_read_skb(&transport->xprt, sk, skb);  in xs_local_data_receive()
968 		skb_free_datagram(sk, skb);  in xs_local_data_receive()
989 		struct sock *sk,  in xs_udp_data_read_skb()  argument
1022 		UDPX_INC_STATS_BH(sk, UDP_MIB_INERRORS);  in xs_udp_data_read_skb()
1026 	UDPX_INC_STATS_BH(sk, UDP_MIB_INDATAGRAMS);  in xs_udp_data_read_skb()
1038 	struct sock *sk;  in xs_udp_data_receive()  local
1042 	sk = transport->inet;  in xs_udp_data_receive()
1043 	if (sk == NULL)  in xs_udp_data_receive()
1046 		skb = skb_recv_datagram(sk, 0, 1, &err);  in xs_udp_data_receive()
1049 		xs_udp_data_read_skb(&transport->xprt, sk, skb);  in xs_udp_data_receive()
1050 		skb_free_datagram(sk, skb);  in xs_udp_data_receive()
1068 static void xs_data_ready(struct sock *sk)  in xs_data_ready()  argument
1072 	read_lock_bh(&sk->sk_callback_lock);  in xs_data_ready()
1074 	xprt = xprt_from_sock(sk);  in xs_data_ready()
1080 	read_unlock_bh(&sk->sk_callback_lock);  in xs_data_ready()
1456 	struct sock *sk;  in xs_tcp_data_receive()  local
1465 	sk = transport->inet;  in xs_tcp_data_receive()
1466 	if (sk == NULL)  in xs_tcp_data_receive()
1471 		lock_sock(sk);  in xs_tcp_data_receive()
1472 		read = tcp_read_sock(sk, &rd_desc, xs_tcp_data_recv);  in xs_tcp_data_receive()
1473 		release_sock(sk);  in xs_tcp_data_receive()
1496 static void xs_tcp_data_ready(struct sock *sk)  in xs_tcp_data_ready()  argument
1503 	read_lock_bh(&sk->sk_callback_lock);  in xs_tcp_data_ready()
1504 	if (!(xprt = xprt_from_sock(sk)))  in xs_tcp_data_ready()
1516 	read_unlock_bh(&sk->sk_callback_lock);  in xs_tcp_data_ready()
1524 static void xs_tcp_state_change(struct sock *sk)  in xs_tcp_state_change()  argument
1529 	read_lock_bh(&sk->sk_callback_lock);  in xs_tcp_state_change()
1530 	if (!(xprt = xprt_from_sock(sk)))  in xs_tcp_state_change()
1534 			sk->sk_state, xprt_connected(xprt),  in xs_tcp_state_change()
1535 			sock_flag(sk, SOCK_DEAD),  in xs_tcp_state_change()
1536 			sock_flag(sk, SOCK_ZAPPED),  in xs_tcp_state_change()
1537 			sk->sk_shutdown);  in xs_tcp_state_change()
1540 	trace_rpc_socket_state_change(xprt, sk->sk_socket);  in xs_tcp_state_change()
1541 	switch (sk->sk_state) {  in xs_tcp_state_change()
1596 	read_unlock_bh(&sk->sk_callback_lock);  in xs_tcp_state_change()
1599 static void xs_write_space(struct sock *sk)  in xs_write_space()  argument
1604 	if (!sk->sk_socket)  in xs_write_space()
1606 	clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);  in xs_write_space()
1608 	if (unlikely(!(xprt = xprt_from_sock(sk))))  in xs_write_space()
1611 	wq = rcu_dereference(sk->sk_wq);  in xs_write_space()
1630 static void xs_udp_write_space(struct sock *sk)  in xs_udp_write_space()  argument
1632 	read_lock_bh(&sk->sk_callback_lock);  in xs_udp_write_space()
1635 	if (sock_writeable(sk))  in xs_udp_write_space()
1636 		xs_write_space(sk);  in xs_udp_write_space()
1638 	read_unlock_bh(&sk->sk_callback_lock);  in xs_udp_write_space()
1651 static void xs_tcp_write_space(struct sock *sk)  in xs_tcp_write_space()  argument
1653 	read_lock_bh(&sk->sk_callback_lock);  in xs_tcp_write_space()
1656 	if (sk_stream_is_writeable(sk))  in xs_tcp_write_space()
1657 		xs_write_space(sk);  in xs_tcp_write_space()
1659 	read_unlock_bh(&sk->sk_callback_lock);  in xs_tcp_write_space()
1665 	struct sock *sk = transport->inet;  in xs_udp_do_set_buffer_size()  local
1668 		sk->sk_userlocks |= SOCK_RCVBUF_LOCK;  in xs_udp_do_set_buffer_size()
1669 		sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2;  in xs_udp_do_set_buffer_size()
1672 		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;  in xs_udp_do_set_buffer_size()
1673 		sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2;  in xs_udp_do_set_buffer_size()
1674 		sk->sk_write_space(sk);  in xs_udp_do_set_buffer_size()
1862 	struct sock *sk = sock->sk;  in xs_reclassify_socketu()  local
1864 	sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC",  in xs_reclassify_socketu()
1870 	struct sock *sk = sock->sk;  in xs_reclassify_socket4()  local
1872 	sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC",  in xs_reclassify_socket4()
1878 	struct sock *sk = sock->sk;  in xs_reclassify_socket6()  local
1880 	sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC",  in xs_reclassify_socket6()
1886 	WARN_ON_ONCE(sock_owned_by_user(sock->sk));  in xs_reclassify_socket()
1887 	if (sock_owned_by_user(sock->sk))  in xs_reclassify_socket()
1960 		struct sock *sk = sock->sk;  in xs_local_finish_connecting()  local
1962 		write_lock_bh(&sk->sk_callback_lock);  in xs_local_finish_connecting()
1964 		xs_save_old_callbacks(transport, sk);  in xs_local_finish_connecting()
1966 		sk->sk_user_data = xprt;  in xs_local_finish_connecting()
1967 		sk->sk_data_ready = xs_data_ready;  in xs_local_finish_connecting()
1968 		sk->sk_write_space = xs_udp_write_space;  in xs_local_finish_connecting()
1969 		sk->sk_error_report = xs_error_report;  in xs_local_finish_connecting()
1970 		sk->sk_allocation = GFP_NOIO;  in xs_local_finish_connecting()
1976 		transport->inet = sk;  in xs_local_finish_connecting()
1978 		write_unlock_bh(&sk->sk_callback_lock);  in xs_local_finish_connecting()
2146 		struct sock *sk = sock->sk;  in xs_udp_finish_connecting()  local
2148 		write_lock_bh(&sk->sk_callback_lock);  in xs_udp_finish_connecting()
2150 		xs_save_old_callbacks(transport, sk);  in xs_udp_finish_connecting()
2152 		sk->sk_user_data = xprt;  in xs_udp_finish_connecting()
2153 		sk->sk_data_ready = xs_data_ready;  in xs_udp_finish_connecting()
2154 		sk->sk_write_space = xs_udp_write_space;  in xs_udp_finish_connecting()
2155 		sk->sk_allocation = GFP_NOIO;  in xs_udp_finish_connecting()
2161 		transport->inet = sk;  in xs_udp_finish_connecting()
2165 		write_unlock_bh(&sk->sk_callback_lock);  in xs_udp_finish_connecting()
2226 		struct sock *sk = sock->sk;  in xs_tcp_finish_connecting()  local
2248 		write_lock_bh(&sk->sk_callback_lock);  in xs_tcp_finish_connecting()
2250 		xs_save_old_callbacks(transport, sk);  in xs_tcp_finish_connecting()
2252 		sk->sk_user_data = xprt;  in xs_tcp_finish_connecting()
2253 		sk->sk_data_ready = xs_tcp_data_ready;  in xs_tcp_finish_connecting()
2254 		sk->sk_state_change = xs_tcp_state_change;  in xs_tcp_finish_connecting()
2255 		sk->sk_write_space = xs_tcp_write_space;  in xs_tcp_finish_connecting()
2256 		sk->sk_error_report = xs_error_report;  in xs_tcp_finish_connecting()
2257 		sk->sk_allocation = GFP_NOIO;  in xs_tcp_finish_connecting()
2260 		sock_reset_flag(sk, SOCK_LINGER);  in xs_tcp_finish_connecting()
2261 		tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF;  in xs_tcp_finish_connecting()
2267 		transport->inet = sk;  in xs_tcp_finish_connecting()
2269 		write_unlock_bh(&sk->sk_callback_lock);  in xs_tcp_finish_connecting()
2327 			sock->sk->sk_state);  in xs_tcp_setup_socket()