219 static inline struct rpc_xprt *xprt_from_sock(struct sock *sk)  in xprt_from_sock()  argument
221 	return (struct rpc_xprt *) sk->sk_user_data;  in xprt_from_sock()
456 	struct sock *sk = transport->inet;  in xs_nospace()  local
474 			sk->sk_write_pending++;  in xs_nospace()
486 	sk->sk_write_space(sk);  in xs_nospace()
760 static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk)  in xs_save_old_callbacks()  argument
762 	transport->old_data_ready = sk->sk_data_ready;  in xs_save_old_callbacks()
763 	transport->old_state_change = sk->sk_state_change;  in xs_save_old_callbacks()
764 	transport->old_write_space = sk->sk_write_space;  in xs_save_old_callbacks()
765 	transport->old_error_report = sk->sk_error_report;  in xs_save_old_callbacks()
768 static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk)  in xs_restore_old_callbacks()  argument
770 	sk->sk_data_ready = transport->old_data_ready;  in xs_restore_old_callbacks()
771 	sk->sk_state_change = transport->old_state_change;  in xs_restore_old_callbacks()
772 	sk->sk_write_space = transport->old_write_space;  in xs_restore_old_callbacks()
773 	sk->sk_error_report = transport->old_error_report;  in xs_restore_old_callbacks()
798 static void xs_error_report(struct sock *sk)  in xs_error_report()  argument
803 	read_lock_bh(&sk->sk_callback_lock);  in xs_error_report()
804 	if (!(xprt = xprt_from_sock(sk)))  in xs_error_report()
807 	err = -sk->sk_err;  in xs_error_report()
811 	if (sk->sk_state == TCP_CLOSE)  in xs_error_report()
815 	trace_rpc_socket_error(xprt, sk->sk_socket, err);  in xs_error_report()
818 	read_unlock_bh(&sk->sk_callback_lock);  in xs_error_report()
824 	struct sock *sk = transport->inet;  in xs_reset_transport()  local
827 	if (sk == NULL)  in xs_reset_transport()
830 	write_lock_bh(&sk->sk_callback_lock);  in xs_reset_transport()
834 	sk->sk_user_data = NULL;  in xs_reset_transport()
836 	xs_restore_old_callbacks(transport, sk);  in xs_reset_transport()
838 	write_unlock_bh(&sk->sk_callback_lock);  in xs_reset_transport()
909 static void xs_local_data_ready(struct sock *sk)  in xs_local_data_ready()  argument
919 	read_lock_bh(&sk->sk_callback_lock);  in xs_local_data_ready()
921 	xprt = xprt_from_sock(sk);  in xs_local_data_ready()
925 	skb = skb_recv_datagram(sk, 0, 1, &err);  in xs_local_data_ready()
961 	skb_free_datagram(sk, skb);  in xs_local_data_ready()
963 	read_unlock_bh(&sk->sk_callback_lock);  in xs_local_data_ready()
972 static void xs_udp_data_ready(struct sock *sk)  in xs_udp_data_ready()  argument
982 	read_lock_bh(&sk->sk_callback_lock);  in xs_udp_data_ready()
984 	if (!(xprt = xprt_from_sock(sk)))  in xs_udp_data_ready()
987 	if ((skb = skb_recv_datagram(sk, 0, 1, &err)) == NULL)  in xs_udp_data_ready()
1014 		UDPX_INC_STATS_BH(sk, UDP_MIB_INERRORS);  in xs_udp_data_ready()
1018 	UDPX_INC_STATS_BH(sk, UDP_MIB_INDATAGRAMS);  in xs_udp_data_ready()
1026 	skb_free_datagram(sk, skb);  in xs_udp_data_ready()
1028 	read_unlock_bh(&sk->sk_callback_lock);  in xs_udp_data_ready()
1396 static void xs_tcp_data_ready(struct sock *sk)  in xs_tcp_data_ready()  argument
1405 	read_lock_bh(&sk->sk_callback_lock);  in xs_tcp_data_ready()
1406 	if (!(xprt = xprt_from_sock(sk))) {  in xs_tcp_data_ready()
1420 		read = tcp_read_sock(sk, &rd_desc, xs_tcp_data_recv);  in xs_tcp_data_ready()
1426 	read_unlock_bh(&sk->sk_callback_lock);  in xs_tcp_data_ready()
1434 static void xs_tcp_state_change(struct sock *sk)  in xs_tcp_state_change()  argument
1439 	read_lock_bh(&sk->sk_callback_lock);  in xs_tcp_state_change()
1440 	if (!(xprt = xprt_from_sock(sk)))  in xs_tcp_state_change()
1444 			sk->sk_state, xprt_connected(xprt),  in xs_tcp_state_change()
1445 			sock_flag(sk, SOCK_DEAD),  in xs_tcp_state_change()
1446 			sock_flag(sk, SOCK_ZAPPED),  in xs_tcp_state_change()
1447 			sk->sk_shutdown);  in xs_tcp_state_change()
1450 	trace_rpc_socket_state_change(xprt, sk->sk_socket);  in xs_tcp_state_change()
1451 	switch (sk->sk_state) {  in xs_tcp_state_change()
1506 	read_unlock_bh(&sk->sk_callback_lock);  in xs_tcp_state_change()
1509 static void xs_write_space(struct sock *sk)  in xs_write_space()  argument
1514 	if (unlikely(!(sock = sk->sk_socket)))  in xs_write_space()
1518 	if (unlikely(!(xprt = xprt_from_sock(sk))))  in xs_write_space()
1536 static void xs_udp_write_space(struct sock *sk)  in xs_udp_write_space()  argument
1538 	read_lock_bh(&sk->sk_callback_lock);  in xs_udp_write_space()
1541 	if (sock_writeable(sk))  in xs_udp_write_space()
1542 		xs_write_space(sk);  in xs_udp_write_space()
1544 	read_unlock_bh(&sk->sk_callback_lock);  in xs_udp_write_space()
1557 static void xs_tcp_write_space(struct sock *sk)  in xs_tcp_write_space()  argument
1559 	read_lock_bh(&sk->sk_callback_lock);  in xs_tcp_write_space()
1562 	if (sk_stream_is_writeable(sk))  in xs_tcp_write_space()
1563 		xs_write_space(sk);  in xs_tcp_write_space()
1565 	read_unlock_bh(&sk->sk_callback_lock);  in xs_tcp_write_space()
1571 	struct sock *sk = transport->inet;  in xs_udp_do_set_buffer_size()  local
1574 		sk->sk_userlocks |= SOCK_RCVBUF_LOCK;  in xs_udp_do_set_buffer_size()
1575 		sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2;  in xs_udp_do_set_buffer_size()
1578 		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;  in xs_udp_do_set_buffer_size()
1579 		sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2;  in xs_udp_do_set_buffer_size()
1580 		sk->sk_write_space(sk);  in xs_udp_do_set_buffer_size()
1768 	struct sock *sk = sock->sk;  in xs_reclassify_socketu()  local
1770 	sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC",  in xs_reclassify_socketu()
1776 	struct sock *sk = sock->sk;  in xs_reclassify_socket4()  local
1778 	sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC",  in xs_reclassify_socket4()
1784 	struct sock *sk = sock->sk;  in xs_reclassify_socket6()  local
1786 	sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC",  in xs_reclassify_socket6()
1792 	WARN_ON_ONCE(sock_owned_by_user(sock->sk));  in xs_reclassify_socket()
1793 	if (sock_owned_by_user(sock->sk))  in xs_reclassify_socket()
1866 		struct sock *sk = sock->sk;  in xs_local_finish_connecting()  local
1868 		write_lock_bh(&sk->sk_callback_lock);  in xs_local_finish_connecting()
1870 		xs_save_old_callbacks(transport, sk);  in xs_local_finish_connecting()
1872 		sk->sk_user_data = xprt;  in xs_local_finish_connecting()
1873 		sk->sk_data_ready = xs_local_data_ready;  in xs_local_finish_connecting()
1874 		sk->sk_write_space = xs_udp_write_space;  in xs_local_finish_connecting()
1875 		sk->sk_error_report = xs_error_report;  in xs_local_finish_connecting()
1876 		sk->sk_allocation = GFP_ATOMIC;  in xs_local_finish_connecting()
1882 		transport->inet = sk;  in xs_local_finish_connecting()
1884 		write_unlock_bh(&sk->sk_callback_lock);  in xs_local_finish_connecting()
2013 		struct sock *sk = sock->sk;  in xs_udp_finish_connecting()  local
2015 		write_lock_bh(&sk->sk_callback_lock);  in xs_udp_finish_connecting()
2017 		xs_save_old_callbacks(transport, sk);  in xs_udp_finish_connecting()
2019 		sk->sk_user_data = xprt;  in xs_udp_finish_connecting()
2020 		sk->sk_data_ready = xs_udp_data_ready;  in xs_udp_finish_connecting()
2021 		sk->sk_write_space = xs_udp_write_space;  in xs_udp_finish_connecting()
2022 		sk->sk_allocation = GFP_ATOMIC;  in xs_udp_finish_connecting()
2028 		transport->inet = sk;  in xs_udp_finish_connecting()
2032 		write_unlock_bh(&sk->sk_callback_lock);  in xs_udp_finish_connecting()
2072 		struct sock *sk = sock->sk;  in xs_tcp_finish_connecting()  local
2087 		write_lock_bh(&sk->sk_callback_lock);  in xs_tcp_finish_connecting()
2089 		xs_save_old_callbacks(transport, sk);  in xs_tcp_finish_connecting()
2091 		sk->sk_user_data = xprt;  in xs_tcp_finish_connecting()
2092 		sk->sk_data_ready = xs_tcp_data_ready;  in xs_tcp_finish_connecting()
2093 		sk->sk_state_change = xs_tcp_state_change;  in xs_tcp_finish_connecting()
2094 		sk->sk_write_space = xs_tcp_write_space;  in xs_tcp_finish_connecting()
2095 		sk->sk_error_report = xs_error_report;  in xs_tcp_finish_connecting()
2096 		sk->sk_allocation = GFP_ATOMIC;  in xs_tcp_finish_connecting()
2099 		sock_reset_flag(sk, SOCK_LINGER);  in xs_tcp_finish_connecting()
2100 		tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF;  in xs_tcp_finish_connecting()
2106 		transport->inet = sk;  in xs_tcp_finish_connecting()
2108 		write_unlock_bh(&sk->sk_callback_lock);  in xs_tcp_finish_connecting()
2169 			sock->sk->sk_state);  in xs_tcp_setup_socket()