40 static inline unsigned int xfrm_dst_hash(struct net *net,  in xfrm_dst_hash()  argument
46 	return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);  in xfrm_dst_hash()
49 static inline unsigned int xfrm_src_hash(struct net *net,  in xfrm_src_hash()  argument
54 	return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);  in xfrm_src_hash()
58 xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr,  in xfrm_spi_hash()  argument
61 	return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);  in xfrm_spi_hash()
102 	struct net *net = container_of(work, struct net, xfrm.state_hash_work);  in xfrm_hash_resize()  local
108 	nsize = xfrm_hash_new_size(net->xfrm.state_hmask);  in xfrm_hash_resize()
124 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
127 	for (i = net->xfrm.state_hmask; i >= 0; i--)  in xfrm_hash_resize()
128 		xfrm_hash_transfer(net->xfrm.state_bydst+i, ndst, nsrc, nspi,  in xfrm_hash_resize()
131 	odst = net->xfrm.state_bydst;  in xfrm_hash_resize()
132 	osrc = net->xfrm.state_bysrc;  in xfrm_hash_resize()
133 	ospi = net->xfrm.state_byspi;  in xfrm_hash_resize()
134 	ohashmask = net->xfrm.state_hmask;  in xfrm_hash_resize()
136 	net->xfrm.state_bydst = ndst;  in xfrm_hash_resize()
137 	net->xfrm.state_bysrc = nsrc;  in xfrm_hash_resize()
138 	net->xfrm.state_byspi = nspi;  in xfrm_hash_resize()
139 	net->xfrm.state_hmask = nhashmask;  in xfrm_hash_resize()
141 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
358 	struct net *net = container_of(work, struct net, xfrm.state_gc_work);  in xfrm_state_gc_task()  local
364 	hlist_move_list(&net->xfrm.state_gc_list, &gc_list);  in xfrm_state_gc_task()
468 struct xfrm_state *xfrm_state_alloc(struct net *net)  in xfrm_state_alloc()  argument
475 		write_pnet(&x->xs_net, net);  in xfrm_state_alloc()
503 	struct net *net = xs_net(x);  in __xfrm_state_destroy()  local
508 	hlist_add_head(&x->gclist, &net->xfrm.state_gc_list);  in __xfrm_state_destroy()
510 	schedule_work(&net->xfrm.state_gc_work);  in __xfrm_state_destroy()
516 	struct net *net = xs_net(x);  in __xfrm_state_delete()  local
521 		spin_lock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
527 		net->xfrm.state_num--;  in __xfrm_state_delete()
528 		spin_unlock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
556 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)  in xfrm_state_flush_secctx_check()  argument
560 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush_secctx_check()
563 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush_secctx_check()
576 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)  in xfrm_state_flush_secctx_check()  argument
582 int xfrm_state_flush(struct net *net, u8 proto, bool task_valid)  in xfrm_state_flush()  argument
586 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
587 	err = xfrm_state_flush_secctx_check(net, proto, task_valid);  in xfrm_state_flush()
592 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush()
595 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush()
599 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
608 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
617 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
622 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si)  in xfrm_sad_getinfo()  argument
624 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
625 	si->sadcnt = net->xfrm.state_num;  in xfrm_sad_getinfo()
626 	si->sadhcnt = net->xfrm.state_hmask;  in xfrm_sad_getinfo()
628 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
654 static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark,  in __xfrm_state_lookup()  argument
659 	unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);  in __xfrm_state_lookup()
662 	hlist_for_each_entry(x, net->xfrm.state_byspi+h, byspi) {  in __xfrm_state_lookup()
678 static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark,  in __xfrm_state_lookup_byaddr()  argument
683 	unsigned int h = xfrm_src_hash(net, daddr, saddr, family);  in __xfrm_state_lookup_byaddr()
686 	hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {  in __xfrm_state_lookup_byaddr()
705 	struct net *net = xs_net(x);  in __xfrm_state_locate()  local
709 		return __xfrm_state_lookup(net, mark, &x->id.daddr,  in __xfrm_state_locate()
712 		return __xfrm_state_lookup_byaddr(net, mark,  in __xfrm_state_locate()
718 static void xfrm_hash_grow_check(struct net *net, int have_hash_collision)  in xfrm_hash_grow_check()  argument
721 	    (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&  in xfrm_hash_grow_check()
722 	    net->xfrm.state_num > net->xfrm.state_hmask)  in xfrm_hash_grow_check()
723 		schedule_work(&net->xfrm.state_hash_work);  in xfrm_hash_grow_check()
770 	struct net *net = xp_net(pol);  in xfrm_state_find()  local
782 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
783 	h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);  in xfrm_state_find()
784 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_state_find()
799 	h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family);  in xfrm_state_find()
800 	hlist_for_each_entry(x, net->xfrm.state_bydst+h_wildcard, bydst) {  in xfrm_state_find()
817 		    (x0 = __xfrm_state_lookup(net, mark, daddr, tmpl->id.spi,  in xfrm_state_find()
824 		c.net = net;  in xfrm_state_find()
834 		x = xfrm_state_alloc(net);  in xfrm_state_find()
854 			list_add(&x->km.all, &net->xfrm.state_all);  in xfrm_state_find()
855 			hlist_add_head(&x->bydst, net->xfrm.state_bydst+h);  in xfrm_state_find()
856 			h = xfrm_src_hash(net, daddr, saddr, encap_family);  in xfrm_state_find()
857 			hlist_add_head(&x->bysrc, net->xfrm.state_bysrc+h);  in xfrm_state_find()
859 				h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);  in xfrm_state_find()
860 				hlist_add_head(&x->byspi, net->xfrm.state_byspi+h);  in xfrm_state_find()
862 			x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in xfrm_state_find()
863 			tasklet_hrtimer_start(&x->mtimer, ktime_set(net->xfrm.sysctl_acq_expires, 0), HRTIMER_MODE_REL);  in xfrm_state_find()
864 			net->xfrm.state_num++;  in xfrm_state_find()
865 			xfrm_hash_grow_check(net, x->bydst.next != NULL);  in xfrm_state_find()
878 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
885 xfrm_stateonly_find(struct net *net, u32 mark,  in xfrm_stateonly_find()  argument
892 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
893 	h = xfrm_dst_hash(net, daddr, saddr, reqid, family);  in xfrm_stateonly_find()
894 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_stateonly_find()
910 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
917 struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,  in xfrm_state_lookup_byspi()  argument
923 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
924 	list_for_each_entry(w, &net->xfrm.state_all, all) {  in xfrm_state_lookup_byspi()
931 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
934 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
941 	struct net *net = xs_net(x);  in __xfrm_state_insert()  local
944 	list_add(&x->km.all, &net->xfrm.state_all);  in __xfrm_state_insert()
946 	h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,  in __xfrm_state_insert()
948 	hlist_add_head(&x->bydst, net->xfrm.state_bydst+h);  in __xfrm_state_insert()
950 	h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);  in __xfrm_state_insert()
951 	hlist_add_head(&x->bysrc, net->xfrm.state_bysrc+h);  in __xfrm_state_insert()
954 		h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,  in __xfrm_state_insert()
957 		hlist_add_head(&x->byspi, net->xfrm.state_byspi+h);  in __xfrm_state_insert()
964 	net->xfrm.state_num++;  in __xfrm_state_insert()
966 	xfrm_hash_grow_check(net, x->bydst.next != NULL);  in __xfrm_state_insert()
972 	struct net *net = xs_net(xnew);  in __xfrm_state_bump_genids()  local
979 	h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family);  in __xfrm_state_bump_genids()
980 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __xfrm_state_bump_genids()
992 	struct net *net = xs_net(x);  in xfrm_state_insert()  local
994 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
997 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1002 static struct xfrm_state *__find_acq_core(struct net *net,  in __find_acq_core()  argument
1010 	unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family);  in __find_acq_core()
1014 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __find_acq_core()
1033 	x = xfrm_state_alloc(net);  in __find_acq_core()
1062 		x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in __find_acq_core()
1064 		tasklet_hrtimer_start(&x->mtimer, ktime_set(net->xfrm.sysctl_acq_expires, 0), HRTIMER_MODE_REL);  in __find_acq_core()
1065 		list_add(&x->km.all, &net->xfrm.state_all);  in __find_acq_core()
1066 		hlist_add_head(&x->bydst, net->xfrm.state_bydst+h);  in __find_acq_core()
1067 		h = xfrm_src_hash(net, daddr, saddr, family);  in __find_acq_core()
1068 		hlist_add_head(&x->bysrc, net->xfrm.state_bysrc+h);  in __find_acq_core()
1070 		net->xfrm.state_num++;  in __find_acq_core()
1072 		xfrm_hash_grow_check(net, x->bydst.next != NULL);  in __find_acq_core()
1078 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq);
1082 	struct net *net = xs_net(x);  in xfrm_state_add()  local
1093 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1104 		x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq);  in xfrm_state_add()
1113 		x1 = __find_acq_core(net, &x->mark, family, x->props.mode,  in xfrm_state_add()
1122 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1139 	struct net *net = xs_net(orig);  in xfrm_state_clone()  local
1140 	struct xfrm_state *x = xfrm_state_alloc(net);  in xfrm_state_clone()
1220 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net)  in xfrm_migrate_state_find()  argument
1225 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
1228 		h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr,  in xfrm_migrate_state_find()
1230 		hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_migrate_state_find()
1245 		h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr,  in xfrm_migrate_state_find()
1247 		hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {  in xfrm_migrate_state_find()
1261 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
1302 	struct net *net = xs_net(x);  in xfrm_state_update()  local
1306 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1326 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1392 xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi,  in xfrm_state_lookup()  argument
1397 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup()
1398 	x = __xfrm_state_lookup(net, mark, daddr, spi, proto, family);  in xfrm_state_lookup()
1399 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup()
1405 xfrm_state_lookup_byaddr(struct net *net, u32 mark,  in xfrm_state_lookup_byaddr()  argument
1411 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
1412 	x = __xfrm_state_lookup_byaddr(net, mark, daddr, saddr, proto, family);  in xfrm_state_lookup_byaddr()
1413 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
1419 xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid,  in xfrm_find_acq()  argument
1425 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
1426 	x = __find_acq_core(net, mark, family, mode, reqid, proto, daddr, saddr, create);  in xfrm_find_acq()
1427 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
1436 	       unsigned short family, struct net *net)  in xfrm_tmpl_sort()  argument
1443 	spin_lock_bh(&net->xfrm.xfrm_state_lock); /*FIXME*/  in xfrm_tmpl_sort()
1446 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_tmpl_sort()
1458 	struct net *net = xs_net(*src);  in xfrm_state_sort()  local
1463 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_sort()
1466 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_sort()
1475 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)  in __xfrm_find_acq_byseq()  argument
1479 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in __xfrm_find_acq_byseq()
1482 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in __xfrm_find_acq_byseq()
1494 struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq)  in xfrm_find_acq_byseq()  argument
1498 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
1499 	x = __xfrm_find_acq_byseq(net, mark, seq);  in xfrm_find_acq_byseq()
1500 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
1544 	struct net *net = xs_net(x);  in xfrm_alloc_spi()  local
1563 		x0 = xfrm_state_lookup(net, mark, &x->id.daddr, minspi, x->id.proto, x->props.family);  in xfrm_alloc_spi()
1573 			x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family);  in xfrm_alloc_spi()
1582 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
1583 		h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, x->props.family);  in xfrm_alloc_spi()
1584 		hlist_add_head(&x->byspi, net->xfrm.state_byspi+h);  in xfrm_alloc_spi()
1585 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
1614 int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk,  in xfrm_state_walk()  argument
1625 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
1627 		x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);  in xfrm_state_walk()
1630 	list_for_each_entry_from(x, &net->xfrm.state_all, all) {  in xfrm_state_walk()
1651 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
1667 void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net)  in xfrm_state_walk_done()  argument
1674 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
1676 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
1804 int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr)  in km_report()  argument
1813 			ret = km->report(net, proto, sel, addr);  in km_report()
2081 int __net_init xfrm_state_init(struct net *net)  in xfrm_state_init()  argument
2085 	INIT_LIST_HEAD(&net->xfrm.state_all);  in xfrm_state_init()
2089 	net->xfrm.state_bydst = xfrm_hash_alloc(sz);  in xfrm_state_init()
2090 	if (!net->xfrm.state_bydst)  in xfrm_state_init()
2092 	net->xfrm.state_bysrc = xfrm_hash_alloc(sz);  in xfrm_state_init()
2093 	if (!net->xfrm.state_bysrc)  in xfrm_state_init()
2095 	net->xfrm.state_byspi = xfrm_hash_alloc(sz);  in xfrm_state_init()
2096 	if (!net->xfrm.state_byspi)  in xfrm_state_init()
2098 	net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);  in xfrm_state_init()
2100 	net->xfrm.state_num = 0;  in xfrm_state_init()
2101 	INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);  in xfrm_state_init()
2102 	INIT_HLIST_HEAD(&net->xfrm.state_gc_list);  in xfrm_state_init()
2103 	INIT_WORK(&net->xfrm.state_gc_work, xfrm_state_gc_task);  in xfrm_state_init()
2104 	spin_lock_init(&net->xfrm.xfrm_state_lock);  in xfrm_state_init()
2108 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_init()
2110 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_init()
2115 void xfrm_state_fini(struct net *net)  in xfrm_state_fini()  argument
2119 	flush_work(&net->xfrm.state_hash_work);  in xfrm_state_fini()
2120 	xfrm_state_flush(net, IPSEC_PROTO_ANY, false);  in xfrm_state_fini()
2121 	flush_work(&net->xfrm.state_gc_work);  in xfrm_state_fini()
2123 	WARN_ON(!list_empty(&net->xfrm.state_all));  in xfrm_state_fini()
2125 	sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);  in xfrm_state_fini()
2126 	WARN_ON(!hlist_empty(net->xfrm.state_byspi));  in xfrm_state_fini()
2127 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_fini()
2128 	WARN_ON(!hlist_empty(net->xfrm.state_bysrc));  in xfrm_state_fini()
2129 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_fini()
2130 	WARN_ON(!hlist_empty(net->xfrm.state_bydst));  in xfrm_state_fini()
2131 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_fini()