43 u32 sk_ehashfn(const struct sock *sk)  in sk_ehashfn()  argument
46 	if (sk->sk_family == AF_INET6 &&  in sk_ehashfn()
47 	    !ipv6_addr_v4mapped(&sk->sk_v6_daddr))  in sk_ehashfn()
48 		return inet6_ehashfn(sock_net(sk),  in sk_ehashfn()
49 				     &sk->sk_v6_rcv_saddr, sk->sk_num,  in sk_ehashfn()
50 				     &sk->sk_v6_daddr, sk->sk_dport);  in sk_ehashfn()
52 	return inet_ehashfn(sock_net(sk),  in sk_ehashfn()
53 			    sk->sk_rcv_saddr, sk->sk_num,  in sk_ehashfn()
54 			    sk->sk_daddr, sk->sk_dport);  in sk_ehashfn()
91 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,  in inet_bind_hash()  argument
94 	inet_sk(sk)->inet_num = snum;  in inet_bind_hash()
95 	sk_add_bind_node(sk, &tb->owners);  in inet_bind_hash()
97 	inet_csk(sk)->icsk_bind_hash = tb;  in inet_bind_hash()
103 static void __inet_put_port(struct sock *sk)  in __inet_put_port()  argument
105 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in __inet_put_port()
106 	const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,  in __inet_put_port()
112 	tb = inet_csk(sk)->icsk_bind_hash;  in __inet_put_port()
113 	__sk_del_bind_node(sk);  in __inet_put_port()
115 	inet_csk(sk)->icsk_bind_hash = NULL;  in __inet_put_port()
116 	inet_sk(sk)->inet_num = 0;  in __inet_put_port()
121 void inet_put_port(struct sock *sk)  in inet_put_port()  argument
124 	__inet_put_port(sk);  in inet_put_port()
129 int __inet_inherit_port(const struct sock *sk, struct sock *child)  in __inet_inherit_port()  argument
131 	struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;  in __inet_inherit_port()
133 	const int bhash = inet_bhashfn(sock_net(sk), port,  in __inet_inherit_port()
139 	tb = inet_csk(sk)->icsk_bind_hash;  in __inet_inherit_port()
151 			if (net_eq(ib_net(tb), sock_net(sk)) &&  in __inet_inherit_port()
157 						     sock_net(sk), head, port);  in __inet_inherit_port()
171 static inline int compute_score(struct sock *sk, struct net *net,  in compute_score()  argument
176 	struct inet_sock *inet = inet_sk(sk);  in compute_score()
178 	if (net_eq(sock_net(sk), net) && inet->inet_num == hnum &&  in compute_score()
179 			!ipv6_only_sock(sk)) {  in compute_score()
181 		score = sk->sk_family == PF_INET ? 2 : 1;  in compute_score()
187 		if (sk->sk_bound_dev_if) {  in compute_score()
188 			if (sk->sk_bound_dev_if != dif)  in compute_score()
192 		if (sk->sk_incoming_cpu == raw_smp_processor_id())  in compute_score()
212 	struct sock *sk, *result;  in __inet_lookup_listener()  local
223 	sk_nulls_for_each_rcu(sk, node, &ilb->head) {  in __inet_lookup_listener()
224 		score = compute_score(sk, net, hnum, daddr, dif);  in __inet_lookup_listener()
226 			result = sk;  in __inet_lookup_listener()
228 			reuseport = sk->sk_reuseport;  in __inet_lookup_listener()
237 				result = sk;  in __inet_lookup_listener()
263 void sock_gen_put(struct sock *sk)  in sock_gen_put()  argument
265 	if (!atomic_dec_and_test(&sk->sk_refcnt))  in sock_gen_put()
268 	if (sk->sk_state == TCP_TIME_WAIT)  in sock_gen_put()
269 		inet_twsk_free(inet_twsk(sk));  in sock_gen_put()
270 	else if (sk->sk_state == TCP_NEW_SYN_RECV)  in sock_gen_put()
271 		reqsk_free(inet_reqsk(sk));  in sock_gen_put()
273 		sk_free(sk);  in sock_gen_put()
279 	sock_gen_put(skb->sk);  in sock_edemux()
291 	struct sock *sk;  in __inet_lookup_established()  local
302 	sk_nulls_for_each_rcu(sk, node, &head->chain) {  in __inet_lookup_established()
303 		if (sk->sk_hash != hash)  in __inet_lookup_established()
305 		if (likely(INET_MATCH(sk, net, acookie,  in __inet_lookup_established()
307 			if (unlikely(!atomic_inc_not_zero(&sk->sk_refcnt)))  in __inet_lookup_established()
309 			if (unlikely(!INET_MATCH(sk, net, acookie,  in __inet_lookup_established()
311 				sock_gen_put(sk);  in __inet_lookup_established()
325 	sk = NULL;  in __inet_lookup_established()
328 	return sk;  in __inet_lookup_established()
334 				    struct sock *sk, __u16 lport,  in __inet_check_established()  argument
338 	struct inet_sock *inet = inet_sk(sk);  in __inet_check_established()
341 	int dif = sk->sk_bound_dev_if;  in __inet_check_established()
344 	struct net *net = sock_net(sk);  in __inet_check_established()
363 				if (twsk_unique(sk, sk2, twp))  in __inet_check_established()
375 	sk->sk_hash = hash;  in __inet_check_established()
376 	WARN_ON(!sk_unhashed(sk));  in __inet_check_established()
377 	__sk_nulls_add_node_rcu(sk, &head->chain);  in __inet_check_established()
383 	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);  in __inet_check_established()
398 static u32 inet_sk_port_offset(const struct sock *sk)  in inet_sk_port_offset()  argument
400 	const struct inet_sock *inet = inet_sk(sk);  in inet_sk_port_offset()
410 bool inet_ehash_insert(struct sock *sk, struct sock *osk)  in inet_ehash_insert()  argument
412 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in inet_ehash_insert()
418 	WARN_ON_ONCE(!sk_unhashed(sk));  in inet_ehash_insert()
420 	sk->sk_hash = sk_ehashfn(sk);  in inet_ehash_insert()
421 	head = inet_ehash_bucket(hashinfo, sk->sk_hash);  in inet_ehash_insert()
423 	lock = inet_ehash_lockp(hashinfo, sk->sk_hash);  in inet_ehash_insert()
427 		WARN_ON_ONCE(sk->sk_hash != osk->sk_hash);  in inet_ehash_insert()
431 		__sk_nulls_add_node_rcu(sk, list);  in inet_ehash_insert()
436 bool inet_ehash_nolisten(struct sock *sk, struct sock *osk)  in inet_ehash_nolisten()  argument
438 	bool ok = inet_ehash_insert(sk, osk);  in inet_ehash_nolisten()
441 		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);  in inet_ehash_nolisten()
443 		percpu_counter_inc(sk->sk_prot->orphan_count);  in inet_ehash_nolisten()
444 		sk->sk_state = TCP_CLOSE;  in inet_ehash_nolisten()
445 		sock_set_flag(sk, SOCK_DEAD);  in inet_ehash_nolisten()
446 		inet_csk_destroy_sock(sk);  in inet_ehash_nolisten()
452 void __inet_hash(struct sock *sk, struct sock *osk)  in __inet_hash()  argument
454 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in __inet_hash()
457 	if (sk->sk_state != TCP_LISTEN) {  in __inet_hash()
458 		inet_ehash_nolisten(sk, osk);  in __inet_hash()
461 	WARN_ON(!sk_unhashed(sk));  in __inet_hash()
462 	ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];  in __inet_hash()
465 	__sk_nulls_add_node_rcu(sk, &ilb->head);  in __inet_hash()
466 	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);  in __inet_hash()
471 void inet_hash(struct sock *sk)  in inet_hash()  argument
473 	if (sk->sk_state != TCP_CLOSE) {  in inet_hash()
475 		__inet_hash(sk, NULL);  in inet_hash()
481 void inet_unhash(struct sock *sk)  in inet_unhash()  argument
483 	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;  in inet_unhash()
487 	if (sk_unhashed(sk))  in inet_unhash()
490 	if (sk->sk_state == TCP_LISTEN)  in inet_unhash()
491 		lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock;  in inet_unhash()
493 		lock = inet_ehash_lockp(hashinfo, sk->sk_hash);  in inet_unhash()
496 	done = __sk_nulls_del_node_init_rcu(sk);  in inet_unhash()
498 		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);  in inet_unhash()
504 		struct sock *sk, u32 port_offset,  in __inet_hash_connect()  argument
509 	const unsigned short snum = inet_sk(sk)->inet_num;  in __inet_hash_connect()
513 	struct net *net = sock_net(sk);  in __inet_hash_connect()
550 					if (!check_established(death_row, sk,  in __inet_hash_connect()
578 		inet_bind_hash(sk, tb, port);  in __inet_hash_connect()
579 		if (sk_unhashed(sk)) {  in __inet_hash_connect()
580 			inet_sk(sk)->inet_sport = htons(port);  in __inet_hash_connect()
581 			inet_ehash_nolisten(sk, (struct sock *)tw);  in __inet_hash_connect()
595 	tb  = inet_csk(sk)->icsk_bind_hash;  in __inet_hash_connect()
597 	if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {  in __inet_hash_connect()
598 		inet_ehash_nolisten(sk, NULL);  in __inet_hash_connect()
604 		ret = check_established(death_row, sk, snum, NULL);  in __inet_hash_connect()
615 		      struct sock *sk)  in inet_hash_connect()  argument
619 	if (!inet_sk(sk)->inet_num)  in inet_hash_connect()
620 		port_offset = inet_sk_port_offset(sk);  in inet_hash_connect()
621 	return __inet_hash_connect(death_row, sk, port_offset,  in inet_hash_connect()