58 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
59 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
250 	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);  in xfrm_policy_flo_get()  local
252 	if (unlikely(pol->walk.dead))  in xfrm_policy_flo_get()
255 		xfrm_pol_hold(pol);  in xfrm_policy_flo_get()
262 	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);  in xfrm_policy_flo_check()  local
264 	return !pol->walk.dead;  in xfrm_policy_flo_check()
416 	struct xfrm_policy *pol;  in xfrm_dst_hash_transfer()  local
422 	hlist_for_each_entry_safe(pol, tmp, list, bydst) {  in xfrm_dst_hash_transfer()
425 		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);  in xfrm_dst_hash_transfer()
426 		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,  in xfrm_dst_hash_transfer()
427 				pol->family, nhashmask, dbits, sbits);  in xfrm_dst_hash_transfer()
429 			hlist_del(&pol->bydst);  in xfrm_dst_hash_transfer()
430 			hlist_add_head(&pol->bydst, ndsttable+h);  in xfrm_dst_hash_transfer()
435 			hlist_del(&pol->bydst);  in xfrm_dst_hash_transfer()
436 			hlist_add_behind(&pol->bydst, entry0);  in xfrm_dst_hash_transfer()
438 		entry0 = &pol->bydst;  in xfrm_dst_hash_transfer()
451 	struct xfrm_policy *pol;  in xfrm_idx_hash_transfer()  local
453 	hlist_for_each_entry_safe(pol, tmp, list, byidx) {  in xfrm_idx_hash_transfer()
456 		h = __idx_hash(pol->index, nhashmask);  in xfrm_idx_hash_transfer()
457 		hlist_add_head(&pol->byidx, nidxtable+h);  in xfrm_idx_hash_transfer()
581 	struct xfrm_policy *pol;  in xfrm_hash_rebuild()  local
633 		hlist_for_each_entry(pol, chain, bydst) {  in xfrm_hash_rebuild()
634 			if (policy->priority >= pol->priority)  in xfrm_hash_rebuild()
635 				newpos = &pol->bydst;  in xfrm_hash_rebuild()
734 				   struct xfrm_policy *pol)  in xfrm_policy_mark_match()
738 	if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)  in xfrm_policy_mark_match()
741 	if ((mark & pol->mark.m) == pol->mark.v &&  in xfrm_policy_mark_match()
742 	    policy->priority == pol->priority)  in xfrm_policy_mark_match()
751 	struct xfrm_policy *pol;  in xfrm_policy_insert()  local
760 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_insert()
761 		if (pol->type == policy->type &&  in xfrm_policy_insert()
762 		    !selector_cmp(&pol->selector, &policy->selector) &&  in xfrm_policy_insert()
763 		    xfrm_policy_mark_match(policy, pol) &&  in xfrm_policy_insert()
764 		    xfrm_sec_ctx_match(pol->security, policy->security) &&  in xfrm_policy_insert()
770 			delpol = pol;  in xfrm_policy_insert()
771 			if (policy->priority > pol->priority)  in xfrm_policy_insert()
773 		} else if (policy->priority >= pol->priority) {  in xfrm_policy_insert()
774 			newpos = &pol->bydst;  in xfrm_policy_insert()
819 	struct xfrm_policy *pol, *ret;  in xfrm_policy_bysel_ctx()  local
826 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_bysel_ctx()
827 		if (pol->type == type &&  in xfrm_policy_bysel_ctx()
828 		    (mark & pol->mark.m) == pol->mark.v &&  in xfrm_policy_bysel_ctx()
829 		    !selector_cmp(sel, &pol->selector) &&  in xfrm_policy_bysel_ctx()
830 		    xfrm_sec_ctx_match(ctx, pol->security)) {  in xfrm_policy_bysel_ctx()
831 			xfrm_pol_hold(pol);  in xfrm_policy_bysel_ctx()
834 								pol->security);  in xfrm_policy_bysel_ctx()
837 					return pol;  in xfrm_policy_bysel_ctx()
839 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_bysel_ctx()
841 			ret = pol;  in xfrm_policy_bysel_ctx()
856 	struct xfrm_policy *pol, *ret;  in xfrm_policy_byid()  local
867 	hlist_for_each_entry(pol, chain, byidx) {  in xfrm_policy_byid()
868 		if (pol->type == type && pol->index == id &&  in xfrm_policy_byid()
869 		    (mark & pol->mark.m) == pol->mark.v) {  in xfrm_policy_byid()
870 			xfrm_pol_hold(pol);  in xfrm_policy_byid()
873 								pol->security);  in xfrm_policy_byid()
876 					return pol;  in xfrm_policy_byid()
878 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_byid()
880 			ret = pol;  in xfrm_policy_byid()
899 		struct xfrm_policy *pol;  in xfrm_policy_flush_secctx_check()  local
902 		hlist_for_each_entry(pol,  in xfrm_policy_flush_secctx_check()
904 			if (pol->type != type)  in xfrm_policy_flush_secctx_check()
906 			err = security_xfrm_policy_delete(pol->security);  in xfrm_policy_flush_secctx_check()
908 				xfrm_audit_policy_delete(pol, 0, task_valid);  in xfrm_policy_flush_secctx_check()
913 			hlist_for_each_entry(pol,  in xfrm_policy_flush_secctx_check()
916 				if (pol->type != type)  in xfrm_policy_flush_secctx_check()
919 								pol->security);  in xfrm_policy_flush_secctx_check()
921 					xfrm_audit_policy_delete(pol, 0,  in xfrm_policy_flush_secctx_check()
949 		struct xfrm_policy *pol;  in xfrm_policy_flush()  local
953 		hlist_for_each_entry(pol,  in xfrm_policy_flush()
955 			if (pol->type != type)  in xfrm_policy_flush()
957 			__xfrm_policy_unlink(pol, dir);  in xfrm_policy_flush()
961 			xfrm_audit_policy_delete(pol, 1, task_valid);  in xfrm_policy_flush()
963 			xfrm_policy_kill(pol);  in xfrm_policy_flush()
971 			hlist_for_each_entry(pol,  in xfrm_policy_flush()
974 				if (pol->type != type)  in xfrm_policy_flush()
976 				__xfrm_policy_unlink(pol, dir);  in xfrm_policy_flush()
980 				xfrm_audit_policy_delete(pol, 1, task_valid);  in xfrm_policy_flush()
981 				xfrm_policy_kill(pol);  in xfrm_policy_flush()
1001 	struct xfrm_policy *pol;  in xfrm_policy_walk()  local
1022 		pol = container_of(x, struct xfrm_policy, walk);  in xfrm_policy_walk()
1024 		    walk->type != pol->type)  in xfrm_policy_walk()
1026 		error = func(pol, xfrm_policy_id2dir(pol->index),  in xfrm_policy_walk()
1070 static int xfrm_policy_match(const struct xfrm_policy *pol,  in xfrm_policy_match()  argument
1074 	const struct xfrm_selector *sel = &pol->selector;  in xfrm_policy_match()
1078 	if (pol->family != family ||  in xfrm_policy_match()
1079 	    (fl->flowi_mark & pol->mark.m) != pol->mark.v ||  in xfrm_policy_match()
1080 	    pol->type != type)  in xfrm_policy_match()
1085 		ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,  in xfrm_policy_match()
1096 	struct xfrm_policy *pol, *ret;  in xfrm_policy_lookup_bytype()  local
1109 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_lookup_bytype()
1110 		err = xfrm_policy_match(pol, fl, type, family, dir);  in xfrm_policy_lookup_bytype()
1119 			ret = pol;  in xfrm_policy_lookup_bytype()
1125 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_policy_lookup_bytype()
1126 		if ((pol->priority >= priority) && ret)  in xfrm_policy_lookup_bytype()
1129 		err = xfrm_policy_match(pol, fl, type, family, dir);  in xfrm_policy_lookup_bytype()
1138 			ret = pol;  in xfrm_policy_lookup_bytype()
1154 	struct xfrm_policy *pol;  in __xfrm_policy_lookup()  local
1156 	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir);  in __xfrm_policy_lookup()
1157 	if (pol != NULL)  in __xfrm_policy_lookup()
1158 		return pol;  in __xfrm_policy_lookup()
1185 	struct xfrm_policy *pol;  in xfrm_policy_lookup()  local
1190 	pol = __xfrm_policy_lookup(net, fl, family, flow_to_policy_dir(dir));  in xfrm_policy_lookup()
1191 	if (IS_ERR_OR_NULL(pol))  in xfrm_policy_lookup()
1192 		return ERR_CAST(pol);  in xfrm_policy_lookup()
1196 	xfrm_pol_hold(pol);  in xfrm_policy_lookup()
1198 	return &pol->flo;  in xfrm_policy_lookup()
1221 	struct xfrm_policy *pol;  in xfrm_sk_policy_lookup()  local
1226 	pol = rcu_dereference(sk->sk_policy[dir]);  in xfrm_sk_policy_lookup()
1227 	if (pol != NULL) {  in xfrm_sk_policy_lookup()
1228 		bool match = xfrm_selector_match(&pol->selector, fl,  in xfrm_sk_policy_lookup()
1233 			if ((sk->sk_mark & pol->mark.m) != pol->mark.v) {  in xfrm_sk_policy_lookup()
1234 				pol = NULL;  in xfrm_sk_policy_lookup()
1237 			err = security_xfrm_policy_lookup(pol->security,  in xfrm_sk_policy_lookup()
1241 				xfrm_pol_hold(pol);  in xfrm_sk_policy_lookup()
1243 				pol = NULL;  in xfrm_sk_policy_lookup()
1245 				pol = ERR_PTR(err);  in xfrm_sk_policy_lookup()
1247 			pol = NULL;  in xfrm_sk_policy_lookup()
1252 	return pol;  in xfrm_sk_policy_lookup()
1255 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)  in __xfrm_policy_link()  argument
1257 	struct net *net = xp_net(pol);  in __xfrm_policy_link()
1259 	list_add(&pol->walk.all, &net->xfrm.policy_all);  in __xfrm_policy_link()
1261 	xfrm_pol_hold(pol);  in __xfrm_policy_link()
1264 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,  in __xfrm_policy_unlink()  argument
1267 	struct net *net = xp_net(pol);  in __xfrm_policy_unlink()
1269 	if (list_empty(&pol->walk.all))  in __xfrm_policy_unlink()
1273 	if (!hlist_unhashed(&pol->bydst)) {  in __xfrm_policy_unlink()
1274 		hlist_del(&pol->bydst);  in __xfrm_policy_unlink()
1275 		hlist_del(&pol->byidx);  in __xfrm_policy_unlink()
1278 	list_del_init(&pol->walk.all);  in __xfrm_policy_unlink()
1281 	return pol;  in __xfrm_policy_unlink()
1284 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_link()  argument
1286 	__xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_link()
1289 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)  in xfrm_sk_policy_unlink()  argument
1291 	__xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);  in xfrm_sk_policy_unlink()
1294 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)  in xfrm_policy_delete()  argument
1296 	struct net *net = xp_net(pol);  in xfrm_policy_delete()
1299 	pol = __xfrm_policy_unlink(pol, dir);  in xfrm_policy_delete()
1301 	if (pol) {  in xfrm_policy_delete()
1302 		xfrm_policy_kill(pol);  in xfrm_policy_delete()
1309 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)  in xfrm_sk_policy_insert()  argument
1311 	struct net *net = xp_net(pol);  in xfrm_sk_policy_insert()
1315 	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)  in xfrm_sk_policy_insert()
1322 	if (pol) {  in xfrm_sk_policy_insert()
1323 		pol->curlft.add_time = get_seconds();  in xfrm_sk_policy_insert()
1324 		pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);  in xfrm_sk_policy_insert()
1325 		xfrm_sk_policy_link(pol, dir);  in xfrm_sk_policy_insert()
1327 	rcu_assign_pointer(sk->sk_policy[dir], pol);  in xfrm_sk_policy_insert()
1329 		if (pol)  in xfrm_sk_policy_insert()
1330 			xfrm_policy_requeue(old_pol, pol);  in xfrm_sk_policy_insert()
1909 	struct xfrm_policy *pol = (struct xfrm_policy *)arg;  in xfrm_policy_queue_process()  local
1910 	struct net *net = xp_net(pol);  in xfrm_policy_queue_process()
1911 	struct xfrm_policy_queue *pq = &pol->polq;  in xfrm_policy_queue_process()
1939 			xfrm_pol_hold(pol);  in xfrm_policy_queue_process()
1971 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
1977 	xfrm_pol_put(pol);  in xfrm_policy_queue_process()
1985 	struct xfrm_policy *pol = xdst->pols[0];  in xdst_queue_output()  local
1986 	struct xfrm_policy_queue *pq = &pol->polq;  in xdst_queue_output()
2010 		xfrm_pol_put(pol);  in xdst_queue_output()
2015 		xfrm_pol_hold(pol);  in xdst_queue_output()
2466 	struct xfrm_policy *pol;  in __xfrm_policy_check()  local
2500 	pol = NULL;  in __xfrm_policy_check()
2503 		pol = xfrm_sk_policy_lookup(sk, dir, &fl);  in __xfrm_policy_check()
2504 		if (IS_ERR(pol)) {  in __xfrm_policy_check()
2510 	if (!pol) {  in __xfrm_policy_check()
2516 			pol = ERR_CAST(flo);  in __xfrm_policy_check()
2518 			pol = container_of(flo, struct xfrm_policy, flo);  in __xfrm_policy_check()
2521 	if (IS_ERR(pol)) {  in __xfrm_policy_check()
2526 	if (!pol) {  in __xfrm_policy_check()
2535 	pol->curlft.use_time = get_seconds();  in __xfrm_policy_check()
2537 	pols[0] = pol;  in __xfrm_policy_check()
2555 	if (pol->action == XFRM_POLICY_ALLOW) {  in __xfrm_policy_check()
2568 			if (pols[pi] != pol &&  in __xfrm_policy_check()
3178 	struct xfrm_policy *pol, *ret = NULL;  in xfrm_migrate_policy_find()  local
3184 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_migrate_policy_find()
3185 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
3186 		    pol->type == type) {  in xfrm_migrate_policy_find()
3187 			ret = pol;  in xfrm_migrate_policy_find()
3193 	hlist_for_each_entry(pol, chain, bydst) {  in xfrm_migrate_policy_find()
3194 		if ((pol->priority >= priority) && ret)  in xfrm_migrate_policy_find()
3197 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&  in xfrm_migrate_policy_find()
3198 		    pol->type == type) {  in xfrm_migrate_policy_find()
3199 			ret = pol;  in xfrm_migrate_policy_find()
3241 static int xfrm_policy_migrate(struct xfrm_policy *pol,  in xfrm_policy_migrate()  argument
3247 	write_lock_bh(&pol->lock);  in xfrm_policy_migrate()
3248 	if (unlikely(pol->walk.dead)) {  in xfrm_policy_migrate()
3250 		write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
3254 	for (i = 0; i < pol->xfrm_nr; i++) {  in xfrm_policy_migrate()
3256 			if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))  in xfrm_policy_migrate()
3259 			if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&  in xfrm_policy_migrate()
3260 			    pol->xfrm_vec[i].mode != XFRM_MODE_BEET)  in xfrm_policy_migrate()
3263 			memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,  in xfrm_policy_migrate()
3264 			       sizeof(pol->xfrm_vec[i].id.daddr));  in xfrm_policy_migrate()
3265 			memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,  in xfrm_policy_migrate()
3266 			       sizeof(pol->xfrm_vec[i].saddr));  in xfrm_policy_migrate()
3267 			pol->xfrm_vec[i].encap_family = mp->new_family;  in xfrm_policy_migrate()
3269 			atomic_inc(&pol->genid);  in xfrm_policy_migrate()
3273 	write_unlock_bh(&pol->lock);  in xfrm_policy_migrate()
3320 	struct xfrm_policy *pol = NULL;  in xfrm_migrate()  local
3330 	if ((pol = xfrm_migrate_policy_find(sel, dir, type, net)) == NULL) {  in xfrm_migrate()
3351 	if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)  in xfrm_migrate()
3363 	xfrm_pol_put(pol);  in xfrm_migrate()
3370 	if (pol)  in xfrm_migrate()
3371 		xfrm_pol_put(pol);  in xfrm_migrate()