125 		struct svc_sock *svsk =  in svc_release_skb()  local
130 		skb_free_datagram_locked(svsk->sk_sk, skb);  in svc_release_skb()
143 	struct svc_sock *svsk =  in svc_set_cmsg_data()  local
145 	switch (svsk->sk_sk->sk_family) {  in svc_set_cmsg_data()
236 	struct svc_sock	*svsk =  in svc_sendto()  local
238 	struct socket	*sock = svsk->sk_sock;  in svc_sendto()
271 		svsk, xdr->head[0].iov_base, xdr->head[0].iov_len,  in svc_sendto()
280 static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining)  in svc_one_sock_name()  argument
282 	const struct sock *sk = svsk->sk_sk;  in svc_one_sock_name()
320 	struct svc_sock *svsk =  in svc_recvfrom()  local
329 	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_recvfrom()
330 	len = kernel_recvmsg(svsk->sk_sock, &msg, iov, nr, buflen,  in svc_recvfrom()
336 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_recvfrom()
339 		svsk, iov[0].iov_base, iov[0].iov_len, len);  in svc_recvfrom()
407 	struct svc_sock	*svsk = (struct svc_sock *)sk->sk_user_data;  in svc_udp_data_ready()  local
410 	if (svsk) {  in svc_udp_data_ready()
412 			svsk, sk,  in svc_udp_data_ready()
413 			test_bit(XPT_BUSY, &svsk->sk_xprt.xpt_flags));  in svc_udp_data_ready()
414 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_data_ready()
415 		svc_xprt_enqueue(&svsk->sk_xprt);  in svc_udp_data_ready()
426 	struct svc_sock	*svsk = (struct svc_sock *)(sk->sk_user_data);  in svc_write_space()  local
429 	if (svsk) {  in svc_write_space()
431 			svsk, sk, test_bit(XPT_BUSY, &svsk->sk_xprt.xpt_flags));  in svc_write_space()
432 		svc_xprt_enqueue(&svsk->sk_xprt);  in svc_write_space()
437 		       svsk);  in svc_write_space()
444 	struct svc_sock *svsk =	container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_has_wspace()  local
445 	struct svc_serv *serv = svsk->sk_xprt.xpt_server;  in svc_tcp_has_wspace()
451 	if (sk_stream_wspace(svsk->sk_sk) >= required ||  in svc_tcp_has_wspace()
452 	    (sk_stream_min_wspace(svsk->sk_sk) == 0 &&  in svc_tcp_has_wspace()
455 	set_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_tcp_has_wspace()
461 	struct svc_sock *svsk = (struct svc_sock *)(sk->sk_user_data);  in svc_tcp_write_space()  local
466 	if (!svsk || svc_tcp_has_wspace(&svsk->sk_xprt))  in svc_tcp_write_space()
473 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_adjust_wspace()  local
476 		clear_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_tcp_adjust_wspace()
539 	struct svc_sock	*svsk =  in svc_udp_recvfrom()  local
541 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_udp_recvfrom()
557 	if (test_and_clear_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags))  in svc_udp_recvfrom()
566 	    svc_sock_setbufsize(svsk->sk_sock,  in svc_udp_recvfrom()
570 	clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_recvfrom()
572 	err = kernel_recvmsg(svsk->sk_sock, &msg, NULL,  in svc_udp_recvfrom()
575 		skb = skb_recv_datagram(svsk->sk_sk, 0, 1, &err);  in svc_udp_recvfrom()
581 			set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_recvfrom()
592 	svsk->sk_sk->sk_stamp = skb->tstamp;  in svc_udp_recvfrom()
593 	set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); /* there may be more data... */  in svc_udp_recvfrom()
616 		skb_free_datagram_locked(svsk->sk_sk, skb);  in svc_udp_recvfrom()
645 	skb_free_datagram_locked(svsk->sk_sk, skb);  in svc_udp_recvfrom()
668 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_udp_has_wspace()  local
676 	set_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_udp_has_wspace()
677 	required = atomic_read(&svsk->sk_xprt.xpt_reserved) + serv->sv_max_mesg;  in svc_udp_has_wspace()
678 	if (required*2 > sock_wspace(svsk->sk_sk))  in svc_udp_has_wspace()
680 	clear_bit(SOCK_NOSPACE, &svsk->sk_sock->flags);  in svc_udp_has_wspace()
719 static void svc_udp_init(struct svc_sock *svsk, struct svc_serv *serv)  in svc_udp_init()  argument
723 	svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_udp_class,  in svc_udp_init()
724 		      &svsk->sk_xprt, serv);  in svc_udp_init()
725 	clear_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags);  in svc_udp_init()
726 	svsk->sk_sk->sk_data_ready = svc_udp_data_ready;  in svc_udp_init()
727 	svsk->sk_sk->sk_write_space = svc_write_space;  in svc_udp_init()
733 	svc_sock_setbufsize(svsk->sk_sock,  in svc_udp_init()
734 			    3 * svsk->sk_xprt.xpt_server->sv_max_mesg,  in svc_udp_init()
735 			    3 * svsk->sk_xprt.xpt_server->sv_max_mesg);  in svc_udp_init()
738 	set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_udp_init()
739 	set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags);  in svc_udp_init()
742 	switch (svsk->sk_sk->sk_family) {  in svc_udp_init()
754 	err = kernel_setsockopt(svsk->sk_sock, level, optname,  in svc_udp_init()
765 	struct svc_sock	*svsk = (struct svc_sock *)sk->sk_user_data;  in svc_tcp_listen_data_ready()  local
782 		if (svsk) {  in svc_tcp_listen_data_ready()
783 			set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_listen_data_ready()
784 			svc_xprt_enqueue(&svsk->sk_xprt);  in svc_tcp_listen_data_ready()
799 	struct svc_sock	*svsk = (struct svc_sock *)sk->sk_user_data;  in svc_tcp_state_change()  local
805 	if (!svsk)  in svc_tcp_state_change()
808 		set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags);  in svc_tcp_state_change()
809 		svc_xprt_enqueue(&svsk->sk_xprt);  in svc_tcp_state_change()
817 	struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data;  in svc_tcp_data_ready()  local
822 	if (svsk) {  in svc_tcp_data_ready()
823 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_data_ready()
824 		svc_xprt_enqueue(&svsk->sk_xprt);  in svc_tcp_data_ready()
835 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_accept()  local
838 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_tcp_accept()
839 	struct socket	*sock = svsk->sk_sock;  in svc_tcp_accept()
845 	dprintk("svc: tcp_accept %p sock %p\n", svsk, sock);  in svc_tcp_accept()
849 	clear_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_accept()
860 	set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_accept()
912 static unsigned int svc_tcp_restore_pages(struct svc_sock *svsk, struct svc_rqst *rqstp)  in svc_tcp_restore_pages()  argument
916 	if (svsk->sk_datalen == 0)  in svc_tcp_restore_pages()
918 	len = svsk->sk_datalen;  in svc_tcp_restore_pages()
923 		BUG_ON(svsk->sk_pages[i] == NULL);  in svc_tcp_restore_pages()
924 		rqstp->rq_pages[i] = svsk->sk_pages[i];  in svc_tcp_restore_pages()
925 		svsk->sk_pages[i] = NULL;  in svc_tcp_restore_pages()
931 static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp)  in svc_tcp_save_pages()  argument
935 	if (svsk->sk_datalen == 0)  in svc_tcp_save_pages()
937 	len = svsk->sk_datalen;  in svc_tcp_save_pages()
940 		svsk->sk_pages[i] = rqstp->rq_pages[i];  in svc_tcp_save_pages()
945 static void svc_tcp_clear_pages(struct svc_sock *svsk)  in svc_tcp_clear_pages()  argument
949 	if (svsk->sk_datalen == 0)  in svc_tcp_clear_pages()
951 	len = svsk->sk_datalen;  in svc_tcp_clear_pages()
954 		if (svsk->sk_pages[i] == NULL) {  in svc_tcp_clear_pages()
958 		put_page(svsk->sk_pages[i]);  in svc_tcp_clear_pages()
959 		svsk->sk_pages[i] = NULL;  in svc_tcp_clear_pages()
962 	svsk->sk_tcplen = 0;  in svc_tcp_clear_pages()
963 	svsk->sk_datalen = 0;  in svc_tcp_clear_pages()
970 static int svc_tcp_recv_record(struct svc_sock *svsk, struct svc_rqst *rqstp)  in svc_tcp_recv_record()  argument
972 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_tcp_recv_record()
976 	if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) {  in svc_tcp_recv_record()
979 		want = sizeof(rpc_fraghdr) - svsk->sk_tcplen;  in svc_tcp_recv_record()
980 		iov.iov_base = ((char *) &svsk->sk_reclen) + svsk->sk_tcplen;  in svc_tcp_recv_record()
984 		svsk->sk_tcplen += len;  in svc_tcp_recv_record()
992 		dprintk("svc: TCP record, %d bytes\n", svc_sock_reclen(svsk));  in svc_tcp_recv_record()
993 		if (svc_sock_reclen(svsk) + svsk->sk_datalen >  in svc_tcp_recv_record()
996 					svc_sock_reclen(svsk));  in svc_tcp_recv_record()
1001 	return svc_sock_reclen(svsk);  in svc_tcp_recv_record()
1006 	set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags);  in svc_tcp_recv_record()
1010 static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp)  in receive_cb_reply()  argument
1012 	struct rpc_xprt *bc_xprt = svsk->sk_xprt.xpt_bc_xprt;  in receive_cb_reply()
1069 static void svc_tcp_fragment_received(struct svc_sock *svsk)  in svc_tcp_fragment_received()  argument
1073 		svc_sock_final_rec(svsk) ? "final" : "nonfinal",  in svc_tcp_fragment_received()
1074 		svc_sock_reclen(svsk));  in svc_tcp_fragment_received()
1075 	svsk->sk_tcplen = 0;  in svc_tcp_fragment_received()
1076 	svsk->sk_reclen = 0;  in svc_tcp_fragment_received()
1084 	struct svc_sock	*svsk =  in svc_tcp_recvfrom()  local
1086 	struct svc_serv	*serv = svsk->sk_xprt.xpt_server;  in svc_tcp_recvfrom()
1095 		svsk, test_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags),  in svc_tcp_recvfrom()
1096 		test_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags),  in svc_tcp_recvfrom()
1097 		test_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags));  in svc_tcp_recvfrom()
1099 	len = svc_tcp_recv_record(svsk, rqstp);  in svc_tcp_recvfrom()
1103 	base = svc_tcp_restore_pages(svsk, rqstp);  in svc_tcp_recvfrom()
1104 	want = svc_sock_reclen(svsk) - (svsk->sk_tcplen - sizeof(rpc_fraghdr));  in svc_tcp_recvfrom()
1109 						svsk->sk_datalen + want);  in svc_tcp_recvfrom()
1117 		svsk->sk_tcplen += len;  in svc_tcp_recvfrom()
1118 		svsk->sk_datalen += len;  in svc_tcp_recvfrom()
1120 	if (len != want || !svc_sock_final_rec(svsk)) {  in svc_tcp_recvfrom()
1121 		svc_tcp_save_pages(svsk, rqstp);  in svc_tcp_recvfrom()
1125 			svc_tcp_fragment_received(svsk);  in svc_tcp_recvfrom()
1128 				(int)(svsk->sk_tcplen - sizeof(rpc_fraghdr)),  in svc_tcp_recvfrom()
1129 				svc_sock_reclen(svsk));  in svc_tcp_recvfrom()
1133 	if (svsk->sk_datalen < 8) {  in svc_tcp_recvfrom()
1134 		svsk->sk_datalen = 0;  in svc_tcp_recvfrom()
1138 	rqstp->rq_arg.len = svsk->sk_datalen;  in svc_tcp_recvfrom()
1148 	if (test_bit(XPT_LOCAL, &svsk->sk_xprt.xpt_flags))  in svc_tcp_recvfrom()
1156 		len = receive_cb_reply(svsk, rqstp);  in svc_tcp_recvfrom()
1159 	svsk->sk_datalen = 0;  in svc_tcp_recvfrom()
1160 	svc_tcp_fragment_received(svsk);  in svc_tcp_recvfrom()
1165 	svc_xprt_copy_addrs(rqstp, &svsk->sk_xprt);  in svc_tcp_recvfrom()
1178 	       svsk->sk_xprt.xpt_server->sv_name, -len);  in svc_tcp_recvfrom()
1179 	set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags);  in svc_tcp_recvfrom()
1322 static void svc_tcp_init(struct svc_sock *svsk, struct svc_serv *serv)  in svc_tcp_init()  argument
1324 	struct sock	*sk = svsk->sk_sk;  in svc_tcp_init()
1326 	svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_tcp_class,  in svc_tcp_init()
1327 		      &svsk->sk_xprt, serv);  in svc_tcp_init()
1328 	set_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1331 		set_bit(XPT_LISTENER, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1333 		set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1340 		svsk->sk_reclen = 0;  in svc_tcp_init()
1341 		svsk->sk_tcplen = 0;  in svc_tcp_init()
1342 		svsk->sk_datalen = 0;  in svc_tcp_init()
1343 		memset(&svsk->sk_pages[0], 0, sizeof(svsk->sk_pages));  in svc_tcp_init()
1347 		set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1349 			set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags);  in svc_tcp_init()
1359 	struct svc_sock *svsk;  in svc_sock_update_bufs()  local
1362 	list_for_each_entry(svsk, &serv->sv_permsocks, sk_xprt.xpt_list)  in svc_sock_update_bufs()
1363 		set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags);  in svc_sock_update_bufs()
1376 	struct svc_sock	*svsk;  in svc_setup_socket()  local
1382 	svsk = kzalloc(sizeof(*svsk), GFP_KERNEL);  in svc_setup_socket()
1383 	if (!svsk)  in svc_setup_socket()
1395 		kfree(svsk);  in svc_setup_socket()
1399 	inet->sk_user_data = svsk;  in svc_setup_socket()
1400 	svsk->sk_sock = sock;  in svc_setup_socket()
1401 	svsk->sk_sk = inet;  in svc_setup_socket()
1402 	svsk->sk_ostate = inet->sk_state_change;  in svc_setup_socket()
1403 	svsk->sk_odata = inet->sk_data_ready;  in svc_setup_socket()
1404 	svsk->sk_owspace = inet->sk_write_space;  in svc_setup_socket()
1408 		svc_udp_init(svsk, serv);  in svc_setup_socket()
1413 		svc_sock_setbufsize(svsk->sk_sock, 4 * serv->sv_max_mesg,  in svc_setup_socket()
1415 		svc_tcp_init(svsk, serv);  in svc_setup_socket()
1419 				svsk, svsk->sk_sk);  in svc_setup_socket()
1421 	return svsk;  in svc_setup_socket()
1456 	struct svc_sock *svsk = NULL;  in svc_addsock()  local
1476 	svsk = svc_setup_socket(serv, so, SVC_SOCK_DEFAULTS);  in svc_addsock()
1477 	if (IS_ERR(svsk)) {  in svc_addsock()
1479 		err = PTR_ERR(svsk);  in svc_addsock()
1482 	if (kernel_getsockname(svsk->sk_sock, sin, &salen) == 0)  in svc_addsock()
1483 		svc_xprt_set_local(&svsk->sk_xprt, sin, salen);  in svc_addsock()
1484 	svc_add_new_perm_xprt(serv, &svsk->sk_xprt);  in svc_addsock()
1485 	return svc_one_sock_name(svsk, name_return, len);  in svc_addsock()
1501 	struct svc_sock	*svsk;  in svc_create_socket()  local
1566 	svsk = svc_setup_socket(serv, sock, flags);  in svc_create_socket()
1567 	if (IS_ERR(svsk)) {  in svc_create_socket()
1568 		error = PTR_ERR(svsk);  in svc_create_socket()
1571 	svc_xprt_set_local(&svsk->sk_xprt, newsin, newlen);  in svc_create_socket()
1572 	return (struct svc_xprt *)svsk;  in svc_create_socket()
1585 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_sock_detach()  local
1586 	struct sock *sk = svsk->sk_sk;  in svc_sock_detach()
1589 	dprintk("svc: svc_sock_detach(%p)\n", svsk);  in svc_sock_detach()
1592 	sk->sk_state_change = svsk->sk_ostate;  in svc_sock_detach()
1593 	sk->sk_data_ready = svsk->sk_odata;  in svc_sock_detach()
1594 	sk->sk_write_space = svsk->sk_owspace;  in svc_sock_detach()
1606 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_tcp_sock_detach()  local
1608 	dprintk("svc: svc_tcp_sock_detach(%p)\n", svsk);  in svc_tcp_sock_detach()
1613 		svc_tcp_clear_pages(svsk);  in svc_tcp_sock_detach()
1614 		kernel_sock_shutdown(svsk->sk_sock, SHUT_RDWR);  in svc_tcp_sock_detach()
1623 	struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt);  in svc_sock_free()  local
1624 	dprintk("svc: svc_sock_free(%p)\n", svsk);  in svc_sock_free()
1626 	if (svsk->sk_sock->file)  in svc_sock_free()
1627 		sockfd_put(svsk->sk_sock);  in svc_sock_free()
1629 		sock_release(svsk->sk_sock);  in svc_sock_free()
1630 	kfree(svsk);  in svc_sock_free()
1643 	struct svc_sock *svsk;  in svc_bc_create_socket()  local
1652 	svsk = kzalloc(sizeof(*svsk), GFP_KERNEL);  in svc_bc_create_socket()
1653 	if (!svsk)  in svc_bc_create_socket()
1656 	xprt = &svsk->sk_xprt;  in svc_bc_create_socket()