219 static inline struct rpc_xprt *xprt_from_sock(struct sock *sk)  in xprt_from_sock()
326 static int xs_send_kvec(struct socket *sock, struct sockaddr *addr, int addrlen, struct kvec *vec, …  in xs_send_kvec()  argument
339 		return kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len);  in xs_send_kvec()
340 	return kernel_sendmsg(sock, &msg, NULL, 0, 0);  in xs_send_kvec()
343 static int xs_send_pagedata(struct socket *sock, struct xdr_buf *xdr, unsigned int base, int more, …  in xs_send_pagedata()  argument
345 	ssize_t (*do_sendpage)(struct socket *sock, struct page *page,  in xs_send_pagedata()
355 	do_sendpage = sock->ops->sendpage;  in xs_send_pagedata()
365 		err = do_sendpage(sock, *ppage, base, len, flags);  in xs_send_pagedata()
390 static int xs_sendpages(struct socket *sock, struct sockaddr *addr, int addrlen, struct xdr_buf *xd…  in xs_sendpages()  argument
396 	if (unlikely(!sock))  in xs_sendpages()
399 	clear_bit(SOCK_ASYNC_NOSPACE, &sock->flags);  in xs_sendpages()
408 		err = xs_send_kvec(sock, addr, addrlen, &xdr->head[0], base, remainder != 0);  in xs_sendpages()
419 		err = xs_send_pagedata(sock, xdr, base, remainder != 0, zerocopy, &sent);  in xs_sendpages()
429 	err = xs_send_kvec(sock, NULL, 0, &xdr->tail[0], base, 0);  in xs_sendpages()
443 	clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);  in xs_nospace_callback()
456 	struct sock *sk = transport->inet;  in xs_nospace()
468 		if (test_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags)) {  in xs_nospace()
473 			set_bit(SOCK_NOSPACE, &transport->sock->flags);  in xs_nospace()
479 		clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);  in xs_nospace()
526 	status = xs_sendpages(transport->sock, NULL, 0, xdr, req->rq_bytes_sent,  in xs_local_send_request()
582 	status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen,  in xs_udp_send_request()
619 		clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);  in xs_udp_send_request()
635 	struct socket *sock = transport->sock;  in xs_tcp_shutdown()  local
637 	if (sock != NULL) {  in xs_tcp_shutdown()
638 		kernel_sock_shutdown(sock, SHUT_RDWR);  in xs_tcp_shutdown()
639 		trace_rpc_socket_shutdown(xprt, sock);  in xs_tcp_shutdown()
684 		status = xs_sendpages(transport->sock, NULL, 0, xdr,  in xs_tcp_send_request()
725 		clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);  in xs_tcp_send_request()
760 static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk)  in xs_save_old_callbacks()
768 static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk)  in xs_restore_old_callbacks()
798 static void xs_error_report(struct sock *sk)  in xs_error_report()
823 	struct socket *sock = transport->sock;  in xs_reset_transport()  local
824 	struct sock *sk = transport->inet;  in xs_reset_transport()
832 	transport->sock = NULL;  in xs_reset_transport()
841 	trace_rpc_socket_close(xprt, sock);  in xs_reset_transport()
842 	sock_release(sock);  in xs_reset_transport()
909 static void xs_local_data_ready(struct sock *sk)  in xs_local_data_ready()
972 static void xs_udp_data_ready(struct sock *sk)  in xs_udp_data_ready()
1396 static void xs_tcp_data_ready(struct sock *sk)  in xs_tcp_data_ready()
1434 static void xs_tcp_state_change(struct sock *sk)  in xs_tcp_state_change()
1509 static void xs_write_space(struct sock *sk)  in xs_write_space()
1511 	struct socket *sock;  in xs_write_space()  local
1514 	if (unlikely(!(sock = sk->sk_socket)))  in xs_write_space()
1516 	clear_bit(SOCK_NOSPACE, &sock->flags);  in xs_write_space()
1520 	if (test_and_clear_bit(SOCK_ASYNC_NOSPACE, &sock->flags) == 0)  in xs_write_space()
1536 static void xs_udp_write_space(struct sock *sk)  in xs_udp_write_space()
1557 static void xs_tcp_write_space(struct sock *sk)  in xs_tcp_write_space()
1571 	struct sock *sk = transport->inet;  in xs_udp_do_set_buffer_size()
1631 static void xs_sock_set_reuseport(struct socket *sock)  in xs_sock_set_reuseport()  argument
1635 	kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEPORT,  in xs_sock_set_reuseport()
1639 static unsigned short xs_sock_getport(struct socket *sock)  in xs_sock_getport()  argument
1645 	if (kernel_getsockname(sock, (struct sockaddr *)&buf, &buflen) < 0)  in xs_sock_getport()
1672 static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock)  in xs_set_srcport()  argument
1675 		transport->srcport = xs_sock_getport(sock);  in xs_set_srcport()
1697 static int xs_bind(struct sock_xprt *transport, struct socket *sock)  in xs_bind()  argument
1725 		err = kernel_bind(sock, (struct sockaddr *)&myaddr,  in xs_bind()
1766 static inline void xs_reclassify_socketu(struct socket *sock)  in xs_reclassify_socketu()  argument
1768 	struct sock *sk = sock->sk;  in xs_reclassify_socketu()
1774 static inline void xs_reclassify_socket4(struct socket *sock)  in xs_reclassify_socket4()  argument
1776 	struct sock *sk = sock->sk;  in xs_reclassify_socket4()
1782 static inline void xs_reclassify_socket6(struct socket *sock)  in xs_reclassify_socket6()  argument
1784 	struct sock *sk = sock->sk;  in xs_reclassify_socket6()
1790 static inline void xs_reclassify_socket(int family, struct socket *sock)  in xs_reclassify_socket()  argument
1792 	WARN_ON_ONCE(sock_owned_by_user(sock->sk));  in xs_reclassify_socket()
1793 	if (sock_owned_by_user(sock->sk))  in xs_reclassify_socket()
1798 		xs_reclassify_socketu(sock);  in xs_reclassify_socket()
1801 		xs_reclassify_socket4(sock);  in xs_reclassify_socket()
1804 		xs_reclassify_socket6(sock);  in xs_reclassify_socket()
1809 static inline void xs_reclassify_socketu(struct socket *sock)  in xs_reclassify_socketu()  argument
1813 static inline void xs_reclassify_socket4(struct socket *sock)  in xs_reclassify_socket4()  argument
1817 static inline void xs_reclassify_socket6(struct socket *sock)  in xs_reclassify_socket6()  argument
1821 static inline void xs_reclassify_socket(int family, struct socket *sock)  in xs_reclassify_socket()  argument
1834 	struct socket *sock;  in xs_create_sock()  local
1837 	err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1);  in xs_create_sock()
1843 	xs_reclassify_socket(family, sock);  in xs_create_sock()
1846 		xs_sock_set_reuseport(sock);  in xs_create_sock()
1848 	err = xs_bind(transport, sock);  in xs_create_sock()
1850 		sock_release(sock);  in xs_create_sock()
1854 	return sock;  in xs_create_sock()
1860 				      struct socket *sock)  in xs_local_finish_connecting()  argument
1866 		struct sock *sk = sock->sk;  in xs_local_finish_connecting()
1881 		transport->sock = sock;  in xs_local_finish_connecting()
1890 	return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0);  in xs_local_finish_connecting()
1902 	struct socket *sock;  in xs_local_setup_socket()  local
1906 					SOCK_STREAM, 0, &sock, 1);  in xs_local_setup_socket()
1912 	xs_reclassify_socketu(sock);  in xs_local_setup_socket()
1917 	status = xs_local_finish_connecting(xprt, sock);  in xs_local_setup_socket()
1918 	trace_rpc_socket_connect(xprt, sock, status);  in xs_local_setup_socket()
2008 static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)  in xs_udp_finish_connecting()  argument
2013 		struct sock *sk = sock->sk;  in xs_udp_finish_connecting()
2027 		transport->sock = sock;  in xs_udp_finish_connecting()
2042 	struct socket *sock = transport->sock;  in xs_udp_setup_socket()  local
2045 	sock = xs_create_sock(xprt, transport,  in xs_udp_setup_socket()
2048 	if (IS_ERR(sock))  in xs_udp_setup_socket()
2057 	xs_udp_finish_connecting(xprt, sock);  in xs_udp_setup_socket()
2058 	trace_rpc_socket_connect(xprt, sock, 0);  in xs_udp_setup_socket()
2066 static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)  in xs_tcp_finish_connecting()  argument
2072 		struct sock *sk = sock->sk;  in xs_tcp_finish_connecting()
2078 		kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE,  in xs_tcp_finish_connecting()
2080 		kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE,  in xs_tcp_finish_connecting()
2082 		kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL,  in xs_tcp_finish_connecting()
2084 		kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT,  in xs_tcp_finish_connecting()
2105 		transport->sock = sock;  in xs_tcp_finish_connecting()
2120 	ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK);  in xs_tcp_finish_connecting()
2123 		xs_set_srcport(transport, sock);  in xs_tcp_finish_connecting()
2145 	struct socket *sock = transport->sock;  in xs_tcp_setup_socket()  local
2149 	if (!sock) {  in xs_tcp_setup_socket()
2150 		sock = xs_create_sock(xprt, transport,  in xs_tcp_setup_socket()
2153 		if (IS_ERR(sock)) {  in xs_tcp_setup_socket()
2154 			status = PTR_ERR(sock);  in xs_tcp_setup_socket()
2165 	status = xs_tcp_finish_connecting(xprt, sock);  in xs_tcp_setup_socket()
2166 	trace_rpc_socket_connect(xprt, sock, status);  in xs_tcp_setup_socket()
2169 			sock->sk->sk_state);  in xs_tcp_setup_socket()
2225 	if (transport->sock != NULL) {  in xs_connect()
2381 	struct socket *sock = transport->sock;  in bc_sendto()  local
2389 	len = svc_send_common(sock, xbufp,  in bc_sendto()
2879 	transport->sock = bc_sock->sk_sock;  in xs_setup_bc_tcp()