79 int ipv6_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2)  in ipv6_rcv_saddr_equal()  argument
83 	int addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);  in ipv6_rcv_saddr_equal()
89 			(!sk->sk_rcv_saddr || !sk2->sk_rcv_saddr ||  in ipv6_rcv_saddr_equal()
90 			  sk->sk_rcv_saddr == sk2->sk_rcv_saddr));  in ipv6_rcv_saddr_equal()
97 	    !(ipv6_only_sock(sk) && addr_type2 == IPV6_ADDR_MAPPED))  in ipv6_rcv_saddr_equal()
101 	    ipv6_addr_equal(&sk->sk_v6_rcv_saddr, sk2_rcv_saddr6))  in ipv6_rcv_saddr_equal()
123 int udp_v6_get_port(struct sock *sk, unsigned short snum)  in udp_v6_get_port()  argument
126 		udp6_portaddr_hash(sock_net(sk), &in6addr_any, snum);  in udp_v6_get_port()
128 		udp6_portaddr_hash(sock_net(sk), &sk->sk_v6_rcv_saddr, 0);  in udp_v6_get_port()
131 	udp_sk(sk)->udp_portaddr_hash = hash2_partial;  in udp_v6_get_port()
132 	return udp_lib_get_port(sk, snum, ipv6_rcv_saddr_equal, hash2_nulladdr);  in udp_v6_get_port()
135 static void udp_v6_rehash(struct sock *sk)  in udp_v6_rehash()  argument
137 	u16 new_hash = udp6_portaddr_hash(sock_net(sk),  in udp_v6_rehash()
138 					  &sk->sk_v6_rcv_saddr,  in udp_v6_rehash()
139 					  inet_sk(sk)->inet_num);  in udp_v6_rehash()
141 	udp_lib_rehash(sk, new_hash);  in udp_v6_rehash()
144 static inline int compute_score(struct sock *sk, struct net *net,  in compute_score()  argument
153 	if (!net_eq(sock_net(sk), net) ||  in compute_score()
154 	    udp_sk(sk)->udp_port_hash != hnum ||  in compute_score()
155 	    sk->sk_family != PF_INET6)  in compute_score()
159 	inet = inet_sk(sk);  in compute_score()
167 	if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) {  in compute_score()
168 		if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))  in compute_score()
173 	if (!ipv6_addr_any(&sk->sk_v6_daddr)) {  in compute_score()
174 		if (!ipv6_addr_equal(&sk->sk_v6_daddr, saddr))  in compute_score()
179 	if (sk->sk_bound_dev_if) {  in compute_score()
180 		if (sk->sk_bound_dev_if != dif)  in compute_score()
185 	if (sk->sk_incoming_cpu == raw_smp_processor_id())  in compute_score()
191 static inline int compute_score2(struct sock *sk, struct net *net,  in compute_score2()  argument
199 	if (!net_eq(sock_net(sk), net) ||  in compute_score2()
200 	    udp_sk(sk)->udp_port_hash != hnum ||  in compute_score2()
201 	    sk->sk_family != PF_INET6)  in compute_score2()
204 	if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))  in compute_score2()
208 	inet = inet_sk(sk);  in compute_score2()
216 	if (!ipv6_addr_any(&sk->sk_v6_daddr)) {  in compute_score2()
217 		if (!ipv6_addr_equal(&sk->sk_v6_daddr, saddr))  in compute_score2()
222 	if (sk->sk_bound_dev_if) {  in compute_score2()
223 		if (sk->sk_bound_dev_if != dif)  in compute_score2()
228 	if (sk->sk_incoming_cpu == raw_smp_processor_id())  in compute_score2()
240 	struct sock *sk, *result;  in udp6_lib_lookup2()  local
248 	udp_portaddr_for_each_entry_rcu(sk, node, &hslot2->head) {  in udp6_lib_lookup2()
249 		score = compute_score2(sk, net, saddr, sport,  in udp6_lib_lookup2()
252 			result = sk;  in udp6_lib_lookup2()
254 			reuseport = sk->sk_reuseport;  in udp6_lib_lookup2()
263 				result = sk;  in udp6_lib_lookup2()
292 	struct sock *sk, *result;  in __udp6_lib_lookup()  local
328 	sk_nulls_for_each_rcu(sk, node, &hslot->head) {  in __udp6_lib_lookup()
329 		score = compute_score(sk, net, hnum, saddr, sport, daddr, dport, dif);  in __udp6_lib_lookup()
331 			result = sk;  in __udp6_lib_lookup()
333 			reuseport = sk->sk_reuseport;  in __udp6_lib_lookup()
342 				result = sk;  in __udp6_lib_lookup()
372 	struct sock *sk;  in __udp6_lib_lookup_skb()  local
375 	sk = skb_steal_sock(skb);  in __udp6_lib_lookup_skb()
376 	if (unlikely(sk))  in __udp6_lib_lookup_skb()
377 		return sk;  in __udp6_lib_lookup_skb()
395 int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,  in udpv6_recvmsg()  argument
398 	struct ipv6_pinfo *np = inet6_sk(sk);  in udpv6_recvmsg()
399 	struct inet_sock *inet = inet_sk(sk);  in udpv6_recvmsg()
404 	int is_udplite = IS_UDPLITE(sk);  in udpv6_recvmsg()
409 		return ipv6_recv_error(sk, msg, len, addr_len);  in udpv6_recvmsg()
412 		return ipv6_recv_rxpmtu(sk, msg, len, addr_len);  in udpv6_recvmsg()
415 	skb = __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0),  in udpv6_recvmsg()
451 			atomic_inc(&sk->sk_drops);  in udpv6_recvmsg()
453 				UDP_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
457 				UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
465 			UDP_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
468 			UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
472 	sock_recv_ts_and_drops(msg, sk, skb);  in udpv6_recvmsg()
495 		ip6_datagram_recv_common_ctl(sk, msg, skb);  in udpv6_recvmsg()
502 			ip6_datagram_recv_specific_ctl(sk, msg, skb);  in udpv6_recvmsg()
510 	skb_free_datagram_locked(sk, skb);  in udpv6_recvmsg()
515 	slow = lock_sock_fast(sk);  in udpv6_recvmsg()
516 	if (!skb_kill_datagram(sk, skb, flags)) {  in udpv6_recvmsg()
518 			UDP_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
520 			UDP_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
523 			UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
525 			UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_recvmsg()
529 	unlock_sock_fast(sk, slow);  in udpv6_recvmsg()
546 	struct sock *sk;  in __udp6_lib_err()  local
550 	sk = __udp6_lib_lookup(net, daddr, uh->dest,  in __udp6_lib_err()
552 	if (!sk) {  in __udp6_lib_err()
559 		if (!ip6_sk_accept_pmtu(sk))  in __udp6_lib_err()
561 		ip6_sk_update_pmtu(skb, sk, info);  in __udp6_lib_err()
564 		ip6_sk_redirect(skb, sk);  in __udp6_lib_err()
568 	np = inet6_sk(sk);  in __udp6_lib_err()
573 	if (sk->sk_state != TCP_ESTABLISHED && !np->recverr)  in __udp6_lib_err()
577 		ipv6_icmp_error(sk, skb, err, uh->dest, ntohl(info), (u8 *)(uh+1));  in __udp6_lib_err()
579 	sk->sk_err = err;  in __udp6_lib_err()
580 	sk->sk_error_report(sk);  in __udp6_lib_err()
582 	sock_put(sk);  in __udp6_lib_err()
585 static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)  in __udpv6_queue_rcv_skb()  argument
589 	if (!ipv6_addr_any(&sk->sk_v6_daddr)) {  in __udpv6_queue_rcv_skb()
590 		sock_rps_save_rxhash(sk, skb);  in __udpv6_queue_rcv_skb()
591 		sk_mark_napi_id(sk, skb);  in __udpv6_queue_rcv_skb()
592 		sk_incoming_cpu_update(sk);  in __udpv6_queue_rcv_skb()
595 	rc = sock_queue_rcv_skb(sk, skb);  in __udpv6_queue_rcv_skb()
597 		int is_udplite = IS_UDPLITE(sk);  in __udpv6_queue_rcv_skb()
601 			UDP6_INC_STATS_BH(sock_net(sk),  in __udpv6_queue_rcv_skb()
603 		UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);  in __udpv6_queue_rcv_skb()
625 int udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)  in udpv6_queue_rcv_skb()  argument
627 	struct udp_sock *up = udp_sk(sk);  in udpv6_queue_rcv_skb()
629 	int is_udplite = IS_UDPLITE(sk);  in udpv6_queue_rcv_skb()
631 	if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))  in udpv6_queue_rcv_skb()
635 		int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);  in udpv6_queue_rcv_skb()
657 			ret = encap_rcv(sk, skb);  in udpv6_queue_rcv_skb()
659 				UDP_INC_STATS_BH(sock_net(sk),  in udpv6_queue_rcv_skb()
686 	if (rcu_access_pointer(sk->sk_filter)) {  in udpv6_queue_rcv_skb()
691 	if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {  in udpv6_queue_rcv_skb()
692 		UDP6_INC_STATS_BH(sock_net(sk),  in udpv6_queue_rcv_skb()
699 	bh_lock_sock(sk);  in udpv6_queue_rcv_skb()
701 	if (!sock_owned_by_user(sk))  in udpv6_queue_rcv_skb()
702 		rc = __udpv6_queue_rcv_skb(sk, skb);  in udpv6_queue_rcv_skb()
703 	else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {  in udpv6_queue_rcv_skb()
704 		bh_unlock_sock(sk);  in udpv6_queue_rcv_skb()
707 	bh_unlock_sock(sk);  in udpv6_queue_rcv_skb()
712 	UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);  in udpv6_queue_rcv_skb()
714 	UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS, is_udplite);  in udpv6_queue_rcv_skb()
715 	atomic_inc(&sk->sk_drops);  in udpv6_queue_rcv_skb()
720 static bool __udp_v6_is_mcast_sock(struct net *net, struct sock *sk,  in __udp_v6_is_mcast_sock()  argument
725 	struct inet_sock *inet = inet_sk(sk);  in __udp_v6_is_mcast_sock()
727 	if (!net_eq(sock_net(sk), net))  in __udp_v6_is_mcast_sock()
730 	if (udp_sk(sk)->udp_port_hash != hnum ||  in __udp_v6_is_mcast_sock()
731 	    sk->sk_family != PF_INET6 ||  in __udp_v6_is_mcast_sock()
733 	    (!ipv6_addr_any(&sk->sk_v6_daddr) &&  in __udp_v6_is_mcast_sock()
734 		    !ipv6_addr_equal(&sk->sk_v6_daddr, rmt_addr)) ||  in __udp_v6_is_mcast_sock()
735 	    (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif) ||  in __udp_v6_is_mcast_sock()
736 	    (!ipv6_addr_any(&sk->sk_v6_rcv_saddr) &&  in __udp_v6_is_mcast_sock()
737 		    !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, loc_addr)))  in __udp_v6_is_mcast_sock()
739 	if (!inet6_mc_check(sk, loc_addr, rmt_addr))  in __udp_v6_is_mcast_sock()
748 	struct sock *sk;  in flush_stack()  local
752 		sk = stack[i];  in flush_stack()
756 			atomic_inc(&sk->sk_drops);  in flush_stack()
757 			UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_RCVBUFERRORS,  in flush_stack()
758 					  IS_UDPLITE(sk));  in flush_stack()
759 			UDP6_INC_STATS_BH(sock_net(sk), UDP_MIB_INERRORS,  in flush_stack()
760 					  IS_UDPLITE(sk));  in flush_stack()
763 		if (skb1 && udpv6_queue_rcv_skb(sk, skb1) <= 0)  in flush_stack()
765 		sock_put(sk);  in flush_stack()
789 	struct sock *sk, *stack[256 / sizeof(struct sock *)];  in __udp6_lib_mcast_deliver()  local
795 	unsigned int count = 0, offset = offsetof(typeof(*sk), sk_nulls_node);  in __udp6_lib_mcast_deliver()
805 		offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);  in __udp6_lib_mcast_deliver()
809 	sk_nulls_for_each_entry_offset(sk, node, &hslot->head, offset) {  in __udp6_lib_mcast_deliver()
810 		if (__udp_v6_is_mcast_sock(net, sk,  in __udp6_lib_mcast_deliver()
817 		    (uh->check || udp_sk(sk)->no_check6_rx)) {  in __udp6_lib_mcast_deliver()
823 			stack[count++] = sk;  in __udp6_lib_mcast_deliver()
824 			sock_hold(sk);  in __udp6_lib_mcast_deliver()
851 	struct sock *sk;  in __udp6_lib_rcv()  local
902 	sk = __udp6_lib_lookup_skb(skb, uh->source, uh->dest, udptable);  in __udp6_lib_rcv()
903 	if (sk) {  in __udp6_lib_rcv()
906 		if (!uh->check && !udp_sk(sk)->no_check6_rx) {  in __udp6_lib_rcv()
907 			sock_put(sk);  in __udp6_lib_rcv()
912 		if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))  in __udp6_lib_rcv()
916 		ret = udpv6_queue_rcv_skb(sk, skb);  in __udp6_lib_rcv()
917 		sock_put(sk);  in __udp6_lib_rcv()
966 static void udp_v6_flush_pending_frames(struct sock *sk)  in udp_v6_flush_pending_frames()  argument
968 	struct udp_sock *up = udp_sk(sk);  in udp_v6_flush_pending_frames()
971 		udp_flush_pending_frames(sk);  in udp_v6_flush_pending_frames()
975 		ip6_flush_pending_frames(sk);  in udp_v6_flush_pending_frames()
985 static void udp6_hwcsum_outgoing(struct sock *sk, struct sk_buff *skb,  in udp6_hwcsum_outgoing()  argument
1027 	struct sock *sk = skb->sk;  in udp_v6_send_skb()  local
1030 	int is_udplite = IS_UDPLITE(sk);  in udp_v6_send_skb()
1046 	else if (udp_sk(sk)->no_check6_tx) {   /* UDP csum disabled */  in udp_v6_send_skb()
1050 		udp6_hwcsum_outgoing(sk, skb, &fl6->saddr, &fl6->daddr, len);  in udp_v6_send_skb()
1064 		if (err == -ENOBUFS && !inet6_sk(sk)->recverr) {  in udp_v6_send_skb()
1065 			UDP6_INC_STATS_USER(sock_net(sk),  in udp_v6_send_skb()
1070 		UDP6_INC_STATS_USER(sock_net(sk),  in udp_v6_send_skb()
1075 static int udp_v6_push_pending_frames(struct sock *sk)  in udp_v6_push_pending_frames()  argument
1078 	struct udp_sock  *up = udp_sk(sk);  in udp_v6_push_pending_frames()
1083 		return udp_push_pending_frames(sk);  in udp_v6_push_pending_frames()
1088 	fl6 = inet_sk(sk)->cork.fl.u.ip6;  in udp_v6_push_pending_frames()
1090 	skb = ip6_finish_skb(sk);  in udp_v6_push_pending_frames()
1102 int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)  in udpv6_sendmsg()  argument
1105 	struct udp_sock *up = udp_sk(sk);  in udpv6_sendmsg()
1106 	struct inet_sock *inet = inet_sk(sk);  in udpv6_sendmsg()
1107 	struct ipv6_pinfo *np = inet6_sk(sk);  in udpv6_sendmsg()
1123 	int is_udplite = IS_UDPLITE(sk);  in udpv6_sendmsg()
1148 		if (sk->sk_state != TCP_ESTABLISHED)  in udpv6_sendmsg()
1150 		daddr = &sk->sk_v6_daddr;  in udpv6_sendmsg()
1163 			if (__ipv6_only_sock(sk))  in udpv6_sendmsg()
1165 			return udp_sendmsg(sk, msg, len);  in udpv6_sendmsg()
1170 		return udp_sendmsg(sk, msg, len);  in udpv6_sendmsg()
1184 		lock_sock(sk);  in udpv6_sendmsg()
1187 				release_sock(sk);  in udpv6_sendmsg()
1193 		release_sock(sk);  in udpv6_sendmsg()
1209 				flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);  in udpv6_sendmsg()
1219 		if (sk->sk_state == TCP_ESTABLISHED &&  in udpv6_sendmsg()
1220 		    ipv6_addr_equal(daddr, &sk->sk_v6_daddr))  in udpv6_sendmsg()
1221 			daddr = &sk->sk_v6_daddr;  in udpv6_sendmsg()
1228 		if (sk->sk_state != TCP_ESTABLISHED)  in udpv6_sendmsg()
1232 		daddr = &sk->sk_v6_daddr;  in udpv6_sendmsg()
1238 		fl6.flowi6_oif = sk->sk_bound_dev_if;  in udpv6_sendmsg()
1243 	fl6.flowi6_mark = sk->sk_mark;  in udpv6_sendmsg()
1250 		err = ip6_datagram_send_ctl(sock_net(sk), sk, msg, &fl6, opt,  in udpv6_sendmsg()
1257 			flowlabel = fl6_sock_lookup(sk, fl6.flowlabel);  in udpv6_sendmsg()
1273 	fl6.flowi6_proto = sk->sk_protocol;  in udpv6_sendmsg()
1292 	security_sk_classify_flow(sk, flowi6_to_flowi(&fl6));  in udpv6_sendmsg()
1294 	dst = ip6_sk_dst_lookup_flow(sk, &fl6, final_p);  in udpv6_sendmsg()
1315 		skb = ip6_make_skb(sk, getfrag, msg, ulen,  in udpv6_sendmsg()
1325 	lock_sock(sk);  in udpv6_sendmsg()
1329 		release_sock(sk);  in udpv6_sendmsg()
1342 	err = ip6_append_data(sk, getfrag, msg, ulen,  in udpv6_sendmsg()
1347 		udp_v6_flush_pending_frames(sk);  in udpv6_sendmsg()
1349 		err = udp_v6_push_pending_frames(sk);  in udpv6_sendmsg()
1350 	else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))  in udpv6_sendmsg()
1355 	release_sock(sk);  in udpv6_sendmsg()
1360 			ip6_dst_store(sk, dst,  in udpv6_sendmsg()
1361 				      ipv6_addr_equal(&fl6.daddr, &sk->sk_v6_daddr) ?  in udpv6_sendmsg()
1362 				      &sk->sk_v6_daddr : NULL,  in udpv6_sendmsg()
1387 	if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {  in udpv6_sendmsg()
1388 		UDP6_INC_STATS_USER(sock_net(sk),  in udpv6_sendmsg()
1401 void udpv6_destroy_sock(struct sock *sk)  in udpv6_destroy_sock()  argument
1403 	struct udp_sock *up = udp_sk(sk);  in udpv6_destroy_sock()
1404 	lock_sock(sk);  in udpv6_destroy_sock()
1405 	udp_v6_flush_pending_frames(sk);  in udpv6_destroy_sock()
1406 	release_sock(sk);  in udpv6_destroy_sock()
1409 		void (*encap_destroy)(struct sock *sk);  in udpv6_destroy_sock()
1412 			encap_destroy(sk);  in udpv6_destroy_sock()
1415 	inet6_destroy_sock(sk);  in udpv6_destroy_sock()
1421 int udpv6_setsockopt(struct sock *sk, int level, int optname,  in udpv6_setsockopt()  argument
1425 		return udp_lib_setsockopt(sk, level, optname, optval, optlen,  in udpv6_setsockopt()
1427 	return ipv6_setsockopt(sk, level, optname, optval, optlen);  in udpv6_setsockopt()
1431 int compat_udpv6_setsockopt(struct sock *sk, int level, int optname,  in compat_udpv6_setsockopt()  argument
1435 		return udp_lib_setsockopt(sk, level, optname, optval, optlen,  in compat_udpv6_setsockopt()
1437 	return compat_ipv6_setsockopt(sk, level, optname, optval, optlen);  in compat_udpv6_setsockopt()
1441 int udpv6_getsockopt(struct sock *sk, int level, int optname,  in udpv6_getsockopt()  argument
1445 		return udp_lib_getsockopt(sk, level, optname, optval, optlen);  in udpv6_getsockopt()
1446 	return ipv6_getsockopt(sk, level, optname, optval, optlen);  in udpv6_getsockopt()
1450 int compat_udpv6_getsockopt(struct sock *sk, int level, int optname,  in compat_udpv6_getsockopt()  argument
1454 		return udp_lib_getsockopt(sk, level, optname, optval, optlen);  in compat_udpv6_getsockopt()
1455 	return compat_ipv6_getsockopt(sk, level, optname, optval, optlen);  in compat_udpv6_getsockopt()
1510 void udp_v6_clear_sk(struct sock *sk, int size)  in udp_v6_clear_sk()  argument
1512 	struct inet_sock *inet = inet_sk(sk);  in udp_v6_clear_sk()
1515 	sk_prot_clear_portaddr_nulls(sk, offsetof(struct inet_sock, pinet6));  in udp_v6_clear_sk()