58 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
60 						int dir);
184 	int dir;  in xfrm_policy_timer()  local
191 	dir = xfrm_policy_id2dir(xp->index);  in xfrm_policy_timer()
231 		km_policy_expired(xp, dir, 0, 0);  in xfrm_policy_timer()
243 	if (!xfrm_policy_delete(xp, dir))  in xfrm_policy_timer()
244 		km_policy_expired(xp, dir, 1, 0);  in xfrm_policy_timer()
356 			      unsigned short family, int dir,  in __get_hash_thresh()  argument
361 		*dbits = net->xfrm.policy_bydst[dir].dbits4;  in __get_hash_thresh()
362 		*sbits = net->xfrm.policy_bydst[dir].sbits4;  in __get_hash_thresh()
366 		*dbits = net->xfrm.policy_bydst[dir].dbits6;  in __get_hash_thresh()
367 		*sbits = net->xfrm.policy_bydst[dir].sbits6;  in __get_hash_thresh()
378 					    unsigned short family, int dir)  in policy_hash_bysel()  argument
380 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;  in policy_hash_bysel()
385 	__get_hash_thresh(net, family, dir, &dbits, &sbits);  in policy_hash_bysel()
389 		&net->xfrm.policy_inexact[dir] :  in policy_hash_bysel()
390 		net->xfrm.policy_bydst[dir].table + hash);  in policy_hash_bysel()
396 					     unsigned short family, int dir)  in policy_hash_direct()  argument
398 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;  in policy_hash_direct()
403 	__get_hash_thresh(net, family, dir, &dbits, &sbits);  in policy_hash_direct()
406 	return net->xfrm.policy_bydst[dir].table + hash;  in policy_hash_direct()
413 				   int dir)  in xfrm_dst_hash_transfer()  argument
425 		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);  in xfrm_dst_hash_transfer()
466 static void xfrm_bydst_resize(struct net *net, int dir)  in xfrm_bydst_resize()  argument
468 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;  in xfrm_bydst_resize()
471 	struct hlist_head *odst = net->xfrm.policy_bydst[dir].table;  in xfrm_bydst_resize()
481 		xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);  in xfrm_bydst_resize()
483 	net->xfrm.policy_bydst[dir].table = ndst;  in xfrm_bydst_resize()
484 	net->xfrm.policy_bydst[dir].hmask = nhashmask;  in xfrm_bydst_resize()
516 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)  in xfrm_bydst_should_resize()  argument
518 	unsigned int cnt = net->xfrm.policy_count[dir];  in xfrm_bydst_should_resize()
519 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;  in xfrm_bydst_should_resize()
561 	int dir, total;  in xfrm_hash_resize()  local
566 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {  in xfrm_hash_resize()
567 		if (xfrm_bydst_should_resize(net, dir, &total))  in xfrm_hash_resize()
568 			xfrm_bydst_resize(net, dir);  in xfrm_hash_resize()
587 	int dir;  in xfrm_hash_rebuild()  local
606 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {  in xfrm_hash_rebuild()
607 		INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);  in xfrm_hash_rebuild()
608 		hmask = net->xfrm.policy_bydst[dir].hmask;  in xfrm_hash_rebuild()
609 		odst = net->xfrm.policy_bydst[dir].table;  in xfrm_hash_rebuild()
612 		if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {  in xfrm_hash_rebuild()
614 			net->xfrm.policy_bydst[dir].dbits4 = rbits4;  in xfrm_hash_rebuild()
615 			net->xfrm.policy_bydst[dir].sbits4 = lbits4;  in xfrm_hash_rebuild()
616 			net->xfrm.policy_bydst[dir].dbits6 = rbits6;  in xfrm_hash_rebuild()
617 			net->xfrm.policy_bydst[dir].sbits6 = lbits6;  in xfrm_hash_rebuild()
620 			net->xfrm.policy_bydst[dir].dbits4 = lbits4;  in xfrm_hash_rebuild()
621 			net->xfrm.policy_bydst[dir].sbits4 = rbits4;  in xfrm_hash_rebuild()
622 			net->xfrm.policy_bydst[dir].dbits6 = lbits6;  in xfrm_hash_rebuild()
623 			net->xfrm.policy_bydst[dir].sbits6 = rbits6;  in xfrm_hash_rebuild()
658 static u32 xfrm_gen_index(struct net *net, int dir, u32 index)  in xfrm_gen_index()  argument
669 			idx = (idx_generator | dir);  in xfrm_gen_index()
748 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)  in xfrm_policy_insert()  argument
757 	chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);  in xfrm_policy_insert()
784 	__xfrm_policy_link(policy, dir);  in xfrm_policy_insert()
795 		__xfrm_policy_unlink(delpol, dir);  in xfrm_policy_insert()
797 	policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);  in xfrm_policy_insert()
807 	else if (xfrm_bydst_should_resize(net, dir, NULL))  in xfrm_policy_insert()
815 					  int dir, struct xfrm_selector *sel,  in xfrm_policy_bysel_ctx()  argument
824 	chain = policy_hash_bysel(net, sel, sel->family, dir);  in xfrm_policy_bysel_ctx()
839 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_bysel_ctx()
854 				     int dir, u32 id, int delete, int *err)  in xfrm_policy_byid()  argument
860 	if (xfrm_policy_id2dir(id) != dir)  in xfrm_policy_byid()
878 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_byid()
896 	int dir, err = 0;  in xfrm_policy_flush_secctx_check()  local
898 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {  in xfrm_policy_flush_secctx_check()
903 				     &net->xfrm.policy_inexact[dir], bydst) {  in xfrm_policy_flush_secctx_check()
912 		for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {  in xfrm_policy_flush_secctx_check()
914 					     net->xfrm.policy_bydst[dir].table + i,  in xfrm_policy_flush_secctx_check()
940 	int dir, err = 0, cnt = 0;  in xfrm_policy_flush()  local
948 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {  in xfrm_policy_flush()
954 				     &net->xfrm.policy_inexact[dir], bydst) {  in xfrm_policy_flush()
957 			__xfrm_policy_unlink(pol, dir);  in xfrm_policy_flush()
969 		for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {  in xfrm_policy_flush()
972 					     net->xfrm.policy_bydst[dir].table + i,  in xfrm_policy_flush()
976 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_flush()
1072 			     u8 type, u16 family, int dir)  in xfrm_policy_match()  argument
1086 						  dir);  in xfrm_policy_match()
1093 						     u16 family, u8 dir)  in xfrm_policy_lookup_bytype()  argument
1107 	chain = policy_hash_direct(net, daddr, saddr, family, dir);  in xfrm_policy_lookup_bytype()
1110 		err = xfrm_policy_match(pol, fl, type, family, dir);  in xfrm_policy_lookup_bytype()
1124 	chain = &net->xfrm.policy_inexact[dir];  in xfrm_policy_lookup_bytype()
1129 		err = xfrm_policy_match(pol, fl, type, family, dir);  in xfrm_policy_lookup_bytype()
1151 __xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir)  in __xfrm_policy_lookup()  argument
1156 	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir);  in __xfrm_policy_lookup()
1160 	return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir);  in __xfrm_policy_lookup()
1163 static int flow_to_policy_dir(int dir)  in flow_to_policy_dir()  argument
1168 		return dir;  in flow_to_policy_dir()
1170 	switch (dir) {  in flow_to_policy_dir()
1183 		   u8 dir, struct flow_cache_object *old_obj, void *ctx)  in xfrm_policy_lookup()  argument
1190 	pol = __xfrm_policy_lookup(net, fl, family, flow_to_policy_dir(dir));  in xfrm_policy_lookup()
1201 static inline int policy_to_flow_dir(int dir)  in policy_to_flow_dir()  argument
1206 		return dir;  in policy_to_flow_dir()
1207 	switch (dir) {  in policy_to_flow_dir()
1218 static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,  in xfrm_sk_policy_lookup()  argument
1226 	pol = rcu_dereference(sk->sk_policy[dir]);  in xfrm_sk_policy_lookup()
1239 						      policy_to_flow_dir(dir));  in xfrm_sk_policy_lookup()
1255 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)  in __xfrm_policy_link()  argument
1260 	net->xfrm.policy_count[dir]++;  in __xfrm_policy_link()
1265 						int dir)  in __xfrm_policy_unlink()  argument
1279 	net->xfrm.policy_count[dir]--;  in __xfrm_policy_unlink()
1284 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_link()  argument
1286 	__xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_link()
1289 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_unlink()  argument
1291 	__xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_unlink()
1294 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)  in xfrm_policy_delete()  argument
1299 	pol = __xfrm_policy_unlink(pol, dir);  in xfrm_policy_delete()
1309 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)  in xfrm_sk_policy_insert()  argument
1320 	old_pol = rcu_dereference_protected(sk->sk_policy[dir],  in xfrm_sk_policy_insert()
1324 		pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);  in xfrm_sk_policy_insert()
1325 		xfrm_sk_policy_link(pol, dir);  in xfrm_sk_policy_insert()
1327 	rcu_assign_pointer(sk->sk_policy[dir], pol);  in xfrm_sk_policy_insert()
1335 		xfrm_sk_policy_unlink(old_pol, dir);  in xfrm_sk_policy_insert()
1345 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)  in clone_policy()  argument
1368 		xfrm_sk_policy_link(newp, dir);  in clone_policy()
2082 xfrm_bundle_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir,  in xfrm_bundle_lookup()  argument
2115 					       flow_to_policy_dir(dir));  in xfrm_bundle_lookup()
2214 	u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT);  in xfrm_lookup()  local
2266 		flo = flow_cache_lookup(net, fl, family, dir,  in xfrm_lookup()
2462 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,  in __xfrm_policy_check()  argument
2476 	reverse = dir & ~XFRM_POLICY_MASK;  in __xfrm_policy_check()
2477 	dir &= XFRM_POLICY_MASK;  in __xfrm_policy_check()
2478 	fl_dir = policy_to_flow_dir(dir);  in __xfrm_policy_check()
2502 	if (sk && sk->sk_policy[dir]) {  in __xfrm_policy_check()
2503 		pol = xfrm_sk_policy_lookup(sk, dir, &fl);  in __xfrm_policy_check()
2941 	int dir;  in xfrm_policy_init()  local
2957 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {  in xfrm_policy_init()
2960 		net->xfrm.policy_count[dir] = 0;  in xfrm_policy_init()
2961 		net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;  in xfrm_policy_init()
2962 		INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);  in xfrm_policy_init()
2964 		htab = &net->xfrm.policy_bydst[dir];  in xfrm_policy_init()
2989 	for (dir--; dir >= 0; dir--) {  in xfrm_policy_init()
2992 		htab = &net->xfrm.policy_bydst[dir];  in xfrm_policy_init()
3003 	int dir;  in xfrm_policy_fini()  local
3013 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {  in xfrm_policy_fini()
3016 		WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));  in xfrm_policy_fini()
3018 		htab = &net->xfrm.policy_bydst[dir];  in xfrm_policy_fini()
3176 						    u8 dir, u8 type, struct net *net)  in xfrm_migrate_policy_find()  argument
3183 	chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir);  in xfrm_migrate_policy_find()
3192 	chain = &net->xfrm.policy_inexact[dir];  in xfrm_migrate_policy_find()
3315 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,  in xfrm_migrate()  argument
3330 	if ((pol = xfrm_migrate_policy_find(sel, dir, type, net)) == NULL) {  in xfrm_migrate()
3361 	km_migrate(sel, dir, type, m, num_migrate, k);  in xfrm_migrate()