440 	struct sock_xprt *transport = container_of(task->tk_rqstp->rq_xprt, struct sock_xprt, xprt);  in xs_nospace_callback()  local
442 	transport->inet->sk_write_pending--;  in xs_nospace_callback()
443 	clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);  in xs_nospace_callback()
455 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_nospace()  local
456 	struct sock *sk = transport->inet;  in xs_nospace()
468 		if (test_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags)) {  in xs_nospace()
473 			set_bit(SOCK_NOSPACE, &transport->sock->flags);  in xs_nospace()
479 		clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);  in xs_nospace()
515 	struct sock_xprt *transport =  in xs_local_send_request()  local
526 	status = xs_sendpages(transport->sock, NULL, 0, xdr, req->rq_bytes_sent,  in xs_local_send_request()
571 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_udp_send_request()  local
582 	status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen,  in xs_udp_send_request()
619 		clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);  in xs_udp_send_request()
634 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_tcp_shutdown()  local
635 	struct socket *sock = transport->sock;  in xs_tcp_shutdown()
661 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_tcp_send_request()  local
684 		status = xs_sendpages(transport->sock, NULL, 0, xdr,  in xs_tcp_send_request()
725 		clear_bit(SOCK_ASYNC_NOSPACE, &transport->sock->flags);  in xs_tcp_send_request()
760 static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk)  in xs_save_old_callbacks()  argument
762 	transport->old_data_ready = sk->sk_data_ready;  in xs_save_old_callbacks()
763 	transport->old_state_change = sk->sk_state_change;  in xs_save_old_callbacks()
764 	transport->old_write_space = sk->sk_write_space;  in xs_save_old_callbacks()
765 	transport->old_error_report = sk->sk_error_report;  in xs_save_old_callbacks()
768 static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk)  in xs_restore_old_callbacks()  argument
770 	sk->sk_data_ready = transport->old_data_ready;  in xs_restore_old_callbacks()
771 	sk->sk_state_change = transport->old_state_change;  in xs_restore_old_callbacks()
772 	sk->sk_write_space = transport->old_write_space;  in xs_restore_old_callbacks()
773 	sk->sk_error_report = transport->old_error_report;  in xs_restore_old_callbacks()
821 static void xs_reset_transport(struct sock_xprt *transport)  in xs_reset_transport()  argument
823 	struct socket *sock = transport->sock;  in xs_reset_transport()
824 	struct sock *sk = transport->inet;  in xs_reset_transport()
825 	struct rpc_xprt *xprt = &transport->xprt;  in xs_reset_transport()
831 	transport->inet = NULL;  in xs_reset_transport()
832 	transport->sock = NULL;  in xs_reset_transport()
836 	xs_restore_old_callbacks(transport, sk);  in xs_reset_transport()
857 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_close()  local
861 	xs_reset_transport(transport);  in xs_close()
1042 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_tcp_read_fraghdr()  local
1046 	p = ((char *) &transport->tcp_fraghdr) + transport->tcp_offset;  in xs_tcp_read_fraghdr()
1047 	len = sizeof(transport->tcp_fraghdr) - transport->tcp_offset;  in xs_tcp_read_fraghdr()
1049 	transport->tcp_offset += used;  in xs_tcp_read_fraghdr()
1053 	transport->tcp_reclen = ntohl(transport->tcp_fraghdr);  in xs_tcp_read_fraghdr()
1054 	if (transport->tcp_reclen & RPC_LAST_STREAM_FRAGMENT)  in xs_tcp_read_fraghdr()
1055 		transport->tcp_flags |= TCP_RCV_LAST_FRAG;  in xs_tcp_read_fraghdr()
1057 		transport->tcp_flags &= ~TCP_RCV_LAST_FRAG;  in xs_tcp_read_fraghdr()
1058 	transport->tcp_reclen &= RPC_FRAGMENT_SIZE_MASK;  in xs_tcp_read_fraghdr()
1060 	transport->tcp_flags &= ~TCP_RCV_COPY_FRAGHDR;  in xs_tcp_read_fraghdr()
1061 	transport->tcp_offset = 0;  in xs_tcp_read_fraghdr()
1064 	if (unlikely(transport->tcp_reclen < 8)) {  in xs_tcp_read_fraghdr()
1070 			transport->tcp_reclen);  in xs_tcp_read_fraghdr()
1073 static void xs_tcp_check_fraghdr(struct sock_xprt *transport)  in xs_tcp_check_fraghdr()  argument
1075 	if (transport->tcp_offset == transport->tcp_reclen) {  in xs_tcp_check_fraghdr()
1076 		transport->tcp_flags |= TCP_RCV_COPY_FRAGHDR;  in xs_tcp_check_fraghdr()
1077 		transport->tcp_offset = 0;  in xs_tcp_check_fraghdr()
1078 		if (transport->tcp_flags & TCP_RCV_LAST_FRAG) {  in xs_tcp_check_fraghdr()
1079 			transport->tcp_flags &= ~TCP_RCV_COPY_DATA;  in xs_tcp_check_fraghdr()
1080 			transport->tcp_flags |= TCP_RCV_COPY_XID;  in xs_tcp_check_fraghdr()
1081 			transport->tcp_copied = 0;  in xs_tcp_check_fraghdr()
1086 static inline void xs_tcp_read_xid(struct sock_xprt *transport, struct xdr_skb_reader *desc)  in xs_tcp_read_xid()  argument
1091 	len = sizeof(transport->tcp_xid) - transport->tcp_offset;  in xs_tcp_read_xid()
1093 	p = ((char *) &transport->tcp_xid) + transport->tcp_offset;  in xs_tcp_read_xid()
1095 	transport->tcp_offset += used;  in xs_tcp_read_xid()
1098 	transport->tcp_flags &= ~TCP_RCV_COPY_XID;  in xs_tcp_read_xid()
1099 	transport->tcp_flags |= TCP_RCV_READ_CALLDIR;  in xs_tcp_read_xid()
1100 	transport->tcp_copied = 4;  in xs_tcp_read_xid()
1102 			(transport->tcp_flags & TCP_RPC_REPLY) ? "reply for"  in xs_tcp_read_xid()
1104 			ntohl(transport->tcp_xid));  in xs_tcp_read_xid()
1105 	xs_tcp_check_fraghdr(transport);  in xs_tcp_read_xid()
1108 static inline void xs_tcp_read_calldir(struct sock_xprt *transport,  in xs_tcp_read_calldir()  argument
1121 	offset = transport->tcp_offset - sizeof(transport->tcp_xid);  in xs_tcp_read_calldir()
1122 	len = sizeof(transport->tcp_calldir) - offset;  in xs_tcp_read_calldir()
1124 	p = ((char *) &transport->tcp_calldir) + offset;  in xs_tcp_read_calldir()
1126 	transport->tcp_offset += used;  in xs_tcp_read_calldir()
1129 	transport->tcp_flags &= ~TCP_RCV_READ_CALLDIR;  in xs_tcp_read_calldir()
1134 	switch (ntohl(transport->tcp_calldir)) {  in xs_tcp_read_calldir()
1136 		transport->tcp_flags |= TCP_RCV_COPY_CALLDIR;  in xs_tcp_read_calldir()
1137 		transport->tcp_flags |= TCP_RCV_COPY_DATA;  in xs_tcp_read_calldir()
1138 		transport->tcp_flags |= TCP_RPC_REPLY;  in xs_tcp_read_calldir()
1141 		transport->tcp_flags |= TCP_RCV_COPY_CALLDIR;  in xs_tcp_read_calldir()
1142 		transport->tcp_flags |= TCP_RCV_COPY_DATA;  in xs_tcp_read_calldir()
1143 		transport->tcp_flags &= ~TCP_RPC_REPLY;  in xs_tcp_read_calldir()
1147 		xs_tcp_force_close(&transport->xprt);  in xs_tcp_read_calldir()
1149 	xs_tcp_check_fraghdr(transport);  in xs_tcp_read_calldir()
1156 	struct sock_xprt *transport =  in xs_tcp_read_common()  local
1164 	if (transport->tcp_flags & TCP_RCV_COPY_CALLDIR) {  in xs_tcp_read_common()
1168 		memcpy(rcvbuf->head[0].iov_base + transport->tcp_copied,  in xs_tcp_read_common()
1169 			&transport->tcp_calldir,  in xs_tcp_read_common()
1170 			sizeof(transport->tcp_calldir));  in xs_tcp_read_common()
1171 		transport->tcp_copied += sizeof(transport->tcp_calldir);  in xs_tcp_read_common()
1172 		transport->tcp_flags &= ~TCP_RCV_COPY_CALLDIR;  in xs_tcp_read_common()
1176 	if (len > transport->tcp_reclen - transport->tcp_offset) {  in xs_tcp_read_common()
1179 		len = transport->tcp_reclen - transport->tcp_offset;  in xs_tcp_read_common()
1182 		r = xdr_partial_copy_from_skb(rcvbuf, transport->tcp_copied,  in xs_tcp_read_common()
1187 		r = xdr_partial_copy_from_skb(rcvbuf, transport->tcp_copied,  in xs_tcp_read_common()
1191 		transport->tcp_copied += r;  in xs_tcp_read_common()
1192 		transport->tcp_offset += r;  in xs_tcp_read_common()
1204 		transport->tcp_flags &= ~TCP_RCV_COPY_DATA;  in xs_tcp_read_common()
1206 				ntohl(transport->tcp_xid));  in xs_tcp_read_common()
1209 				xprt, transport->tcp_copied,  in xs_tcp_read_common()
1210 				transport->tcp_offset, transport->tcp_reclen);  in xs_tcp_read_common()
1215 			ntohl(transport->tcp_xid), r);  in xs_tcp_read_common()
1217 			"tcp_reclen = %u\n", xprt, transport->tcp_copied,  in xs_tcp_read_common()
1218 			transport->tcp_offset, transport->tcp_reclen);  in xs_tcp_read_common()
1220 	if (transport->tcp_copied == req->rq_private_buf.buflen)  in xs_tcp_read_common()
1221 		transport->tcp_flags &= ~TCP_RCV_COPY_DATA;  in xs_tcp_read_common()
1222 	else if (transport->tcp_offset == transport->tcp_reclen) {  in xs_tcp_read_common()
1223 		if (transport->tcp_flags & TCP_RCV_LAST_FRAG)  in xs_tcp_read_common()
1224 			transport->tcp_flags &= ~TCP_RCV_COPY_DATA;  in xs_tcp_read_common()
1235 	struct sock_xprt *transport =  in xs_tcp_read_reply()  local
1239 	dprintk("RPC:       read reply XID %08x\n", ntohl(transport->tcp_xid));  in xs_tcp_read_reply()
1243 	req = xprt_lookup_rqst(xprt, transport->tcp_xid);  in xs_tcp_read_reply()
1246 				ntohl(transport->tcp_xid));  in xs_tcp_read_reply()
1253 	if (!(transport->tcp_flags & TCP_RCV_COPY_DATA))  in xs_tcp_read_reply()
1254 		xprt_complete_rqst(req->rq_task, transport->tcp_copied);  in xs_tcp_read_reply()
1271 	struct sock_xprt *transport =  in xs_tcp_read_callback()  local
1277 	req = xprt_lookup_bc_request(xprt, transport->tcp_xid);  in xs_tcp_read_callback()
1288 	if (!(transport->tcp_flags & TCP_RCV_COPY_DATA))  in xs_tcp_read_callback()
1289 		xprt_complete_bc_request(req, transport->tcp_copied);  in xs_tcp_read_callback()
1298 	struct sock_xprt *transport =  in _xs_tcp_read_data()  local
1301 	return (transport->tcp_flags & TCP_RPC_REPLY) ?  in _xs_tcp_read_data()
1320 	struct sock_xprt *transport =  in xs_tcp_read_data()  local
1324 		xs_tcp_check_fraghdr(transport);  in xs_tcp_read_data()
1330 		transport->tcp_flags &= ~TCP_RCV_COPY_DATA;  in xs_tcp_read_data()
1334 static inline void xs_tcp_read_discard(struct sock_xprt *transport, struct xdr_skb_reader *desc)  in xs_tcp_read_discard()  argument
1338 	len = transport->tcp_reclen - transport->tcp_offset;  in xs_tcp_read_discard()
1343 	transport->tcp_offset += len;  in xs_tcp_read_discard()
1345 	xs_tcp_check_fraghdr(transport);  in xs_tcp_read_discard()
1351 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_tcp_data_recv()  local
1360 		trace_xs_tcp_data_recv(transport);  in xs_tcp_data_recv()
1363 		if (transport->tcp_flags & TCP_RCV_COPY_FRAGHDR) {  in xs_tcp_data_recv()
1368 		if (transport->tcp_flags & TCP_RCV_COPY_XID) {  in xs_tcp_data_recv()
1369 			xs_tcp_read_xid(transport, &desc);  in xs_tcp_data_recv()
1373 		if (transport->tcp_flags & TCP_RCV_READ_CALLDIR) {  in xs_tcp_data_recv()
1374 			xs_tcp_read_calldir(transport, &desc);  in xs_tcp_data_recv()
1378 		if (transport->tcp_flags & TCP_RCV_COPY_DATA) {  in xs_tcp_data_recv()
1383 		xs_tcp_read_discard(transport, &desc);  in xs_tcp_data_recv()
1385 	trace_xs_tcp_data_recv(transport);  in xs_tcp_data_recv()
1437 	struct sock_xprt *transport;  in xs_tcp_state_change()  local
1449 	transport = container_of(xprt, struct sock_xprt, xprt);  in xs_tcp_state_change()
1457 			transport->tcp_offset = 0;  in xs_tcp_state_change()
1458 			transport->tcp_reclen = 0;  in xs_tcp_state_change()
1459 			transport->tcp_copied = 0;  in xs_tcp_state_change()
1460 			transport->tcp_flags =  in xs_tcp_state_change()
1463 			clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);  in xs_tcp_state_change()
1501 					&transport->sock_state))  in xs_tcp_state_change()
1570 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_udp_do_set_buffer_size()  local
1571 	struct sock *sk = transport->inet;  in xs_udp_do_set_buffer_size()
1573 	if (transport->rcvsize) {  in xs_udp_do_set_buffer_size()
1575 		sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2;  in xs_udp_do_set_buffer_size()
1577 	if (transport->sndsize) {  in xs_udp_do_set_buffer_size()
1579 		sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2;  in xs_udp_do_set_buffer_size()
1594 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_udp_set_buffer_size()  local
1596 	transport->sndsize = 0;  in xs_udp_set_buffer_size()
1598 		transport->sndsize = sndsize + 1024;  in xs_udp_set_buffer_size()
1599 	transport->rcvsize = 0;  in xs_udp_set_buffer_size()
1601 		transport->rcvsize = rcvsize + 1024;  in xs_udp_set_buffer_size()
1672 static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock)  in xs_set_srcport()  argument
1674 	if (transport->srcport == 0)  in xs_set_srcport()
1675 		transport->srcport = xs_sock_getport(sock);  in xs_set_srcport()
1678 static unsigned short xs_get_srcport(struct sock_xprt *transport)  in xs_get_srcport()  argument
1680 	unsigned short port = transport->srcport;  in xs_get_srcport()
1682 	if (port == 0 && transport->xprt.resvport)  in xs_get_srcport()
1687 static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port)  in xs_next_srcport()  argument
1689 	if (transport->srcport != 0)  in xs_next_srcport()
1690 		transport->srcport = 0;  in xs_next_srcport()
1691 	if (!transport->xprt.resvport)  in xs_next_srcport()
1697 static int xs_bind(struct sock_xprt *transport, struct socket *sock)  in xs_bind()  argument
1701 	unsigned short port = xs_get_srcport(transport);  in xs_bind()
1722 	memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen);  in xs_bind()
1726 				transport->xprt.addrlen);  in xs_bind()
1728 			transport->srcport = port;  in xs_bind()
1732 		port = xs_next_srcport(transport, port);  in xs_bind()
1831 		struct sock_xprt *transport, int family, int type,  in xs_create_sock()  argument
1848 	err = xs_bind(transport, sock);  in xs_create_sock()
1862 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt,  in xs_local_finish_connecting()  local
1865 	if (!transport->inet) {  in xs_local_finish_connecting()
1870 		xs_save_old_callbacks(transport, sk);  in xs_local_finish_connecting()
1881 		transport->sock = sock;  in xs_local_finish_connecting()
1882 		transport->inet = sk;  in xs_local_finish_connecting()
1899 static int xs_local_setup_socket(struct sock_xprt *transport)  in xs_local_setup_socket()  argument
1901 	struct rpc_xprt *xprt = &transport->xprt;  in xs_local_setup_socket()
1948 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_local_connect()  local
1964 	ret = xs_local_setup_socket(transport);  in xs_local_connect()
1972 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt,  in xs_set_memalloc()  local
1976 		sk_set_memalloc(transport->inet);  in xs_set_memalloc()
1987 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt,  in xs_swapper()  local
1996 		sk_clear_memalloc(transport->inet);  in xs_swapper()
2010 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_udp_finish_connecting()  local
2012 	if (!transport->inet) {  in xs_udp_finish_connecting()
2017 		xs_save_old_callbacks(transport, sk);  in xs_udp_finish_connecting()
2027 		transport->sock = sock;  in xs_udp_finish_connecting()
2028 		transport->inet = sk;  in xs_udp_finish_connecting()
2039 	struct sock_xprt *transport =  in xs_udp_setup_socket()  local
2041 	struct rpc_xprt *xprt = &transport->xprt;  in xs_udp_setup_socket()
2042 	struct socket *sock = transport->sock;  in xs_udp_setup_socket()
2045 	sock = xs_create_sock(xprt, transport,  in xs_udp_setup_socket()
2061 	xprt_unlock_connect(xprt, transport);  in xs_udp_setup_socket()
2068 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_tcp_finish_connecting()  local
2071 	if (!transport->inet) {  in xs_tcp_finish_connecting()
2089 		xs_save_old_callbacks(transport, sk);  in xs_tcp_finish_connecting()
2105 		transport->sock = sock;  in xs_tcp_finish_connecting()
2106 		transport->inet = sk;  in xs_tcp_finish_connecting()
2119 	set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);  in xs_tcp_finish_connecting()
2123 		xs_set_srcport(transport, sock);  in xs_tcp_finish_connecting()
2143 	struct sock_xprt *transport =  in xs_tcp_setup_socket()  local
2145 	struct socket *sock = transport->sock;  in xs_tcp_setup_socket()
2146 	struct rpc_xprt *xprt = &transport->xprt;  in xs_tcp_setup_socket()
2150 		sock = xs_create_sock(xprt, transport,  in xs_tcp_setup_socket()
2183 		xprt_unlock_connect(xprt, transport);  in xs_tcp_setup_socket()
2200 	xprt_unlock_connect(xprt, transport);  in xs_tcp_setup_socket()
2221 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_connect()  local
2223 	WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport));  in xs_connect()
2225 	if (transport->sock != NULL) {  in xs_connect()
2231 		xs_reset_transport(transport);  in xs_connect()
2234 				   &transport->connect_worker,  in xs_connect()
2244 				   &transport->connect_worker, 0);  in xs_connect()
2285 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_udp_print_stats()  local
2289 			transport->srcport,  in xs_udp_print_stats()
2309 	struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);  in xs_tcp_print_stats()  local
2317 			transport->srcport,  in xs_tcp_print_stats()
2379 	struct sock_xprt *transport =  in bc_sendto()  local
2381 	struct socket *sock = transport->sock;  in bc_sendto()
2607 	struct sock_xprt *transport;  in xs_setup_local()  local
2615 	transport = container_of(xprt, struct sock_xprt, xprt);  in xs_setup_local()
2628 	INIT_DELAYED_WORK(&transport->connect_worker,  in xs_setup_local()
2641 		ret = ERR_PTR(xs_local_setup_socket(transport));  in xs_setup_local()
2677 	struct sock_xprt *transport;  in xs_setup_udp()  local
2684 	transport = container_of(xprt, struct sock_xprt, xprt);  in xs_setup_udp()
2704 		INIT_DELAYED_WORK(&transport->connect_worker,  in xs_setup_udp()
2712 		INIT_DELAYED_WORK(&transport->connect_worker,  in xs_setup_udp()
2754 	struct sock_xprt *transport;  in xs_setup_tcp()  local
2765 	transport = container_of(xprt, struct sock_xprt, xprt);  in xs_setup_tcp()
2783 		INIT_DELAYED_WORK(&transport->connect_worker,  in xs_setup_tcp()
2791 		INIT_DELAYED_WORK(&transport->connect_worker,  in xs_setup_tcp()
2827 	struct sock_xprt *transport;  in xs_setup_bc_tcp()  local
2835 	transport = container_of(xprt, struct sock_xprt, xprt);  in xs_setup_bc_tcp()
2879 	transport->sock = bc_sock->sk_sock;  in xs_setup_bc_tcp()
2880 	transport->inet = bc_sock->sk_sk;  in xs_setup_bc_tcp()