79 int ipv6_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2)  in ipv6_rcv_saddr_equal()  argument
83 	int addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);  in ipv6_rcv_saddr_equal()
89 			(!sk->sk_rcv_saddr || !sk2->sk_rcv_saddr ||  in ipv6_rcv_saddr_equal()
90 			  sk->sk_rcv_saddr == sk2->sk_rcv_saddr));  in ipv6_rcv_saddr_equal()
97 	    !(ipv6_only_sock(sk) && addr_type2 == IPV6_ADDR_MAPPED))  in ipv6_rcv_saddr_equal()
101 	    ipv6_addr_equal(&sk->sk_v6_rcv_saddr, sk2_rcv_saddr6))  in ipv6_rcv_saddr_equal()
123 int udp_v6_get_port(struct sock *sk, unsigned short snum)  in udp_v6_get_port()  argument
126 		udp6_portaddr_hash(sock_net(sk), &in6addr_any, snum);  in udp_v6_get_port()
128 		udp6_portaddr_hash(sock_net(sk), &sk->sk_v6_rcv_saddr, 0);  in udp_v6_get_port()
131 	udp_sk(sk)->udp_portaddr_hash = hash2_partial;  in udp_v6_get_port()
132 	return udp_lib_get_port(sk, snum, ipv6_rcv_saddr_equal, hash2_nulladdr);  in udp_v6_get_port()
135 static void udp_v6_rehash(struct sock *sk)  in udp_v6_rehash()  argument
137 	u16 new_hash = udp6_portaddr_hash(sock_net(sk),  in udp_v6_rehash()
138 					  &sk->sk_v6_rcv_saddr,  in udp_v6_rehash()
139 					  inet_sk(sk)->inet_num);  in udp_v6_rehash()
141 	udp_lib_rehash(sk, new_hash);  in udp_v6_rehash()
144 static inline int compute_score(struct sock *sk, struct net *net,  in compute_score()  argument
153 	if (!net_eq(sock_net(sk), net) ||  in compute_score()
154 	    udp_sk(sk)->udp_port_hash != hnum ||  in compute_score()
155 	    sk->sk_family != PF_INET6)  in compute_score()
159 	inet = inet_sk(sk);  in compute_score()
167 	if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {  in compute_score()
168 		if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))  in compute_score()
173 	if (!ipv6_addr_any(&sk->sk_v6_daddr)) {  in compute_score()
174 		if (!ipv6_addr_equal(&sk->sk_v6_daddr, saddr))  in compute_score()
179 	if (sk->sk_bound_dev_if) {  in compute_score()
180 		if (sk->sk_bound_dev_if != dif)  in compute_score()
189 static inline int compute_score2(struct sock *sk, struct net *net,  in compute_score2()  argument
197 	if (!net_eq(sock_net(sk), net) ||  in compute_score2()
198 	    udp_sk(sk)->udp_port_hash != hnum ||  in compute_score2()
199 	    sk->sk_family != PF_INET6)  in compute_score2()
202 	if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))  in compute_score2()
206 	inet = inet_sk(sk);  in compute_score2()
214 	if (!ipv6_addr_any(&sk->sk_v6_daddr)) {  in compute_score2()
215 		if (!ipv6_addr_equal(&sk->sk_v6_daddr, saddr))  in compute_score2()
220 	if (sk->sk_bound_dev_if) {  in compute_score2()
221 		if (sk->sk_bound_dev_if != dif)  in compute_score2()
235 	struct sock *sk, *result;  in udp6_lib_lookup2()  local
243 	udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {  in udp6_lib_lookup2()
244 		score = compute_score2(sk, net, saddr, sport,  in udp6_lib_lookup2()
247 			result = sk;  in udp6_lib_lookup2()
249 			reuseport = sk->sk_reuseport;  in udp6_lib_lookup2()
259 				result = sk;  in udp6_lib_lookup2()
289 	struct sock *sk, *result;  in __udp6_lib_lookup()  local
325 	sk_nulls_for_each_rcu(sk, node, &hslot->head) {  in __udp6_lib_lookup()
326 		score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif);  in __udp6_lib_lookup()
328 			result = sk;  in __udp6_lib_lookup()
330 			reuseport = sk->sk_reuseport;  in __udp6_lib_lookup()
339 				result = sk;  in __udp6_lib_lookup()
369 	struct sock *sk;  in __udp6_lib_lookup_skb()  local
372 	sk = skb_steal_sock(skb);  in __udp6_lib_lookup_skb()
373 	if (unlikely(sk))  in __udp6_lib_lookup_skb()
374 		return sk;  in __udp6_lib_lookup_skb()
392 int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,  in udpv6_recvmsg()  argument
395 	struct ipv6_pinfo *np = inet6_sk(sk);  in udpv6_recvmsg()
396 	struct inet_sock *inet = inet_sk(sk);  in udpv6_recvmsg()
401 	int is_udplite = IS_UDPLITE(sk);  in udpv6_recvmsg()
406 		return ipv6_recv_error(sk, msg, len, addr_len);  in udpv6_recvmsg()
409 		return ipv6_recv_rxpmtu(sk, msg, len, addr_len);  in udpv6_recvmsg()
412 	skb = __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0),  in udpv6_recvmsg()
448 			atomic_inc(&sk->sk_drops);  in udpv6_recvmsg()
450 				UDP_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
454 				UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
462 			UDP_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
465 			UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
469 	sock_recv_ts_and_drops(msg, sk, skb);  in udpv6_recvmsg()
492 		ip6_datagram_recv_common_ctl(sk, msg, skb);  in udpv6_recvmsg()
499 			ip6_datagram_recv_specific_ctl(sk, msg, skb);  in udpv6_recvmsg()
507 	skb_free_datagram_locked(sk, skb);  in udpv6_recvmsg()
512 	slow = lock_sock_fast(sk);  in udpv6_recvmsg()
513 	if (!skb_kill_datagram(sk, skb, flags)) {  in udpv6_recvmsg()
515 			UDP_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
517 			UDP_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
520 			UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
522 			UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
526 	unlock_sock_fast(sk, slow);  in udpv6_recvmsg()
543 	struct sock *sk;  in __udp6_lib_err()  local
547 	sk = __udp6_lib_lookup(net, daddr, uh->dest,  in __udp6_lib_err()
549 	if (!sk) {  in __udp6_lib_err()
556 		if (!ip6_sk_accept_pmtu(sk))  in __udp6_lib_err()
558 		ip6_sk_update_pmtu(skb, sk, info);  in __udp6_lib_err()
561 		ip6_sk_redirect(skb, sk);  in __udp6_lib_err()
565 	np = inet6_sk(sk);  in __udp6_lib_err()
570 	if (sk->sk_state != TCP_ESTABLISHED && !np->recverr)  in __udp6_lib_err()
574 		ipv6_icmp_error(sk, skb, err, uh->dest, ntohl(info), (u8 *)(uh+1));  in __udp6_lib_err()
576 	sk->sk_err = err;  in __udp6_lib_err()
577 	sk->sk_error_report(sk);  in __udp6_lib_err()
579 	sock_put(sk);  in __udp6_lib_err()
582 static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)  in __udpv6_queue_rcv_skb()  argument
586 	if (!ipv6_addr_any(&sk->sk_v6_daddr)) {  in __udpv6_queue_rcv_skb()
587 		sock_rps_save_rxhash(sk, skb);  in __udpv6_queue_rcv_skb()
588 		sk_mark_napi_id(sk, skb);  in __udpv6_queue_rcv_skb()
589 		sk_incoming_cpu_update(sk);  in __udpv6_queue_rcv_skb()
592 	rc = sock_queue_rcv_skb(sk, skb);  in __udpv6_queue_rcv_skb()
594 		int is_udplite = IS_UDPLITE(sk);  in __udpv6_queue_rcv_skb()
598 			UDP6_INC_STATS_BH(sock_net(sk),  in __udpv6_queue_rcv_skb()
600 		UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);  in __udpv6_queue_rcv_skb()
622 int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)  in udpv6_queue_rcv_skb()  argument
624 	struct udp_sock *up = udp_sk(sk);  in udpv6_queue_rcv_skb()
626 	int is_udplite = IS_UDPLITE(sk);  in udpv6_queue_rcv_skb()
628 	if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))  in udpv6_queue_rcv_skb()
632 		int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);  in udpv6_queue_rcv_skb()
654 			ret = encap_rcv(sk, skb);  in udpv6_queue_rcv_skb()
656 				UDP_INC_STATS_BH(sock_net(sk),  in udpv6_queue_rcv_skb()
683 	if (rcu_access_pointer(sk->sk_filter)) {  in udpv6_queue_rcv_skb()
688 	if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {  in udpv6_queue_rcv_skb()
689 		UDP6_INC_STATS_BH(sock_net(sk),  in udpv6_queue_rcv_skb()
696 	bh_lock_sock(sk);  in udpv6_queue_rcv_skb()
698 	if (!sock_owned_by_user(sk))  in udpv6_queue_rcv_skb()
699 		rc = __udpv6_queue_rcv_skb(sk, skb);  in udpv6_queue_rcv_skb()
700 	else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {  in udpv6_queue_rcv_skb()
701 		bh_unlock_sock(sk);  in udpv6_queue_rcv_skb()
704 	bh_unlock_sock(sk);  in udpv6_queue_rcv_skb()
709 	UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);  in udpv6_queue_rcv_skb()
711 	UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);  in udpv6_queue_rcv_skb()
712 	atomic_inc(&sk->sk_drops);  in udpv6_queue_rcv_skb()
717 static bool __udp_v6_is_mcast_sock(struct net *net, struct sock *sk,  in __udp_v6_is_mcast_sock()  argument
722 	struct inet_sock *inet = inet_sk(sk);  in __udp_v6_is_mcast_sock()
724 	if (!net_eq(sock_net(sk), net))  in __udp_v6_is_mcast_sock()
727 	if (udp_sk(sk)->udp_port_hash != hnum ||  in __udp_v6_is_mcast_sock()
728 	    sk->sk_family != PF_INET6 ||  in __udp_v6_is_mcast_sock()
730 	    (!ipv6_addr_any(&sk->sk_v6_daddr) &&  in __udp_v6_is_mcast_sock()
731 		    !ipv6_addr_equal(&sk->sk_v6_daddr, rmt_addr)) ||  in __udp_v6_is_mcast_sock()
732 	    (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif) ||  in __udp_v6_is_mcast_sock()
733 	    (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) &&  in __udp_v6_is_mcast_sock()
734 		    !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, loc_addr)))  in __udp_v6_is_mcast_sock()
736 	if (!inet6_mc_check(sk, loc_addr, rmt_addr))  in __udp_v6_is_mcast_sock()
745 	struct sock *sk;  in flush_stack()  local
749 		sk = stack[i];  in flush_stack()
753 			atomic_inc(&sk->sk_drops);  in flush_stack()
754 			UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,  in flush_stack()
755 					  IS_UDPLITE(sk));  in flush_stack()
756 			UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,  in flush_stack()
757 					  IS_UDPLITE(sk));  in flush_stack()
760 		if (skb1 && udpv6_queue_rcv_skb(sk, skb1) <= 0)  in flush_stack()
762 		sock_put(sk);  in flush_stack()
786 	struct sock *sk, *stack[256 / sizeof(struct sock *)];  in __udp6_lib_mcast_deliver()  local
792 	unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);  in __udp6_lib_mcast_deliver()
802 		offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);  in __udp6_lib_mcast_deliver()
806 	sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {  in __udp6_lib_mcast_deliver()
807 		if (__udp_v6_is_mcast_sock(net, sk,  in __udp6_lib_mcast_deliver()
814 		    (uh->check || udp_sk(sk)->no_check6_rx)) {  in __udp6_lib_mcast_deliver()
820 			stack[count++] = sk;  in __udp6_lib_mcast_deliver()
821 			sock_hold(sk);  in __udp6_lib_mcast_deliver()
848 	struct sock *sk;  in __udp6_lib_rcv()  local
899 	sk = __udp6_lib_lookup_skb(skb, uh->source, uh->dest, udptable);  in __udp6_lib_rcv()
900 	if (sk) {  in __udp6_lib_rcv()
903 		if (!uh->check && !udp_sk(sk)->no_check6_rx) {  in __udp6_lib_rcv()
904 			sock_put(sk);  in __udp6_lib_rcv()
909 		if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))  in __udp6_lib_rcv()
913 		ret = udpv6_queue_rcv_skb(sk, skb);  in __udp6_lib_rcv()
914 		sock_put(sk);  in __udp6_lib_rcv()
965 static void udp_v6_flush_pending_frames(struct sock *sk)  in udp_v6_flush_pending_frames()  argument
967 	struct udp_sock *up = udp_sk(sk);  in udp_v6_flush_pending_frames()
970 		udp_flush_pending_frames(sk);  in udp_v6_flush_pending_frames()
974 		ip6_flush_pending_frames(sk);  in udp_v6_flush_pending_frames()
984 static void udp6_hwcsum_outgoing(struct sock *sk, struct sk_buff *skb,  in udp6_hwcsum_outgoing()  argument
1026 	struct sock *sk = skb->sk;  in udp_v6_send_skb()  local
1029 	int is_udplite = IS_UDPLITE(sk);  in udp_v6_send_skb()
1045 	else if (udp_sk(sk)->no_check6_tx) {   /* UDP csum disabled */  in udp_v6_send_skb()
1049 		udp6_hwcsum_outgoing(sk, skb, &fl6->saddr, &fl6->daddr, len);  in udp_v6_send_skb()
1063 		if (err == -ENOBUFS && !inet6_sk(sk)->recverr) {  in udp_v6_send_skb()
1064 			UDP6_INC_STATS_USER(sock_net(sk),  in udp_v6_send_skb()
1069 		UDP6_INC_STATS_USER(sock_net(sk),  in udp_v6_send_skb()
1074 static int udp_v6_push_pending_frames(struct sock *sk)  in udp_v6_push_pending_frames()  argument
1077 	struct udp_sock  *up = udp_sk(sk);  in udp_v6_push_pending_frames()
1082 		return udp_push_pending_frames(sk);  in udp_v6_push_pending_frames()
1087 	fl6 = inet_sk(sk)->cork.fl.u.ip6;  in udp_v6_push_pending_frames()
1089 	skb = ip6_finish_skb(sk);  in udp_v6_push_pending_frames()
1101 int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)  in udpv6_sendmsg()  argument
1104 	struct udp_sock *up = udp_sk(sk);  in udpv6_sendmsg()
1105 	struct inet_sock *inet = inet_sk(sk);  in udpv6_sendmsg()
1106 	struct ipv6_pinfo *np = inet6_sk(sk);  in udpv6_sendmsg()
1122 	int is_udplite = IS_UDPLITE(sk);  in udpv6_sendmsg()
1147 		if (sk->sk_state != TCP_ESTABLISHED)  in udpv6_sendmsg()
1149 		daddr = &sk->sk_v6_daddr;  in udpv6_sendmsg()
1162 			if (__ipv6_only_sock(sk))  in udpv6_sendmsg()
1164 			return udp_sendmsg(sk, msg, len);  in udpv6_sendmsg()
1169 		return udp_sendmsg(sk, msg, len);  in udpv6_sendmsg()
1183 		lock_sock(sk);  in udpv6_sendmsg()
1186 				release_sock(sk);  in udpv6_sendmsg()
1192 		release_sock(sk);  in udpv6_sendmsg()
1208 				flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);  in udpv6_sendmsg()
1218 		if (sk->sk_state == TCP_ESTABLISHED &&  in udpv6_sendmsg()
1219 		    ipv6_addr_equal(daddr, &sk->sk_v6_daddr))  in udpv6_sendmsg()
1220 			daddr = &sk->sk_v6_daddr;  in udpv6_sendmsg()
1227 		if (sk->sk_state != TCP_ESTABLISHED)  in udpv6_sendmsg()
1231 		daddr = &sk->sk_v6_daddr;  in udpv6_sendmsg()
1237 		fl6.flowi6_oif = sk->sk_bound_dev_if;  in udpv6_sendmsg()
1242 	fl6.flowi6_mark = sk->sk_mark;  in udpv6_sendmsg()
1249 		err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, &fl6, opt,  in udpv6_sendmsg()
1256 			flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);  in udpv6_sendmsg()
1272 	fl6.flowi6_proto = sk->sk_protocol;  in udpv6_sendmsg()
1291 	security_sk_classify_flow(sk, flowi6_to_flowi(&fl6));  in udpv6_sendmsg()
1293 	dst = ip6_sk_dst_lookup_flow(sk, &fl6, final_p);  in udpv6_sendmsg()
1314 		skb = ip6_make_skb(sk, getfrag, msg, ulen,  in udpv6_sendmsg()
1324 	lock_sock(sk);  in udpv6_sendmsg()
1328 		release_sock(sk);  in udpv6_sendmsg()
1341 	err = ip6_append_data(sk, getfrag, msg, ulen,  in udpv6_sendmsg()
1346 		udp_v6_flush_pending_frames(sk);  in udpv6_sendmsg()
1348 		err = udp_v6_push_pending_frames(sk);  in udpv6_sendmsg()
1349 	else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))  in udpv6_sendmsg()
1354 	release_sock(sk);  in udpv6_sendmsg()
1359 			ip6_dst_store(sk, dst,  in udpv6_sendmsg()
1360 				      ipv6_addr_equal(&fl6.daddr, &sk->sk_v6_daddr) ?  in udpv6_sendmsg()
1361 				      &sk->sk_v6_daddr : NULL,  in udpv6_sendmsg()
1386 	if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {  in udpv6_sendmsg()
1387 		UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_sendmsg()
1400 void udpv6_destroy_sock(struct sock *sk)  in udpv6_destroy_sock()  argument
1402 	struct udp_sock *up = udp_sk(sk);  in udpv6_destroy_sock()
1403 	lock_sock(sk);  in udpv6_destroy_sock()
1404 	udp_v6_flush_pending_frames(sk);  in udpv6_destroy_sock()
1405 	release_sock(sk);  in udpv6_destroy_sock()
1408 		void (*encap_destroy)(struct sock *sk);  in udpv6_destroy_sock()
1411 			encap_destroy(sk);  in udpv6_destroy_sock()
1414 	inet6_destroy_sock(sk);  in udpv6_destroy_sock()
1420 int udpv6_setsockopt(struct sock *sk, int level, int optname,  in udpv6_setsockopt()  argument
1424 		return udp_lib_setsockopt(sk, level, optname, optval, optlen,  in udpv6_setsockopt()
1426 	return ipv6_setsockopt(sk, level, optname, optval, optlen);  in udpv6_setsockopt()
1430 int compat_udpv6_setsockopt(struct sock *sk, int level, int optname,  in compat_udpv6_setsockopt()  argument
1434 		return udp_lib_setsockopt(sk, level, optname, optval, optlen,  in compat_udpv6_setsockopt()
1436 	return compat_ipv6_setsockopt(sk, level, optname, optval, optlen);  in compat_udpv6_setsockopt()
1440 int udpv6_getsockopt(struct sock *sk, int level, int optname,  in udpv6_getsockopt()  argument
1444 		return udp_lib_getsockopt(sk, level, optname, optval, optlen);  in udpv6_getsockopt()
1445 	return ipv6_getsockopt(sk, level, optname, optval, optlen);  in udpv6_getsockopt()
1449 int compat_udpv6_getsockopt(struct sock *sk, int level, int optname,  in compat_udpv6_getsockopt()  argument
1453 		return udp_lib_getsockopt(sk, level, optname, optval, optlen);  in compat_udpv6_getsockopt()
1454 	return compat_ipv6_getsockopt(sk, level, optname, optval, optlen);  in compat_udpv6_getsockopt()
1508 void udp_v6_clear_sk(struct sock *sk, int size)  in udp_v6_clear_sk()  argument
1510 	struct inet_sock *inet = inet_sk(sk);  in udp_v6_clear_sk()
1513 	sk_prot_clear_portaddr_nulls(sk, offsetof(struct inet_sock, pinet6));  in udp_v6_clear_sk()