62 #define for_nexthops(fi) {						\  argument
64 	for (nhsel = 0, nh = (fi)->fib_nh;				\
65 	     nhsel < (fi)->fib_nhs;					\
68 #define change_nexthops(fi) {						\  argument
70 	for (nhsel = 0,	nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
71 	     nhsel < (fi)->fib_nhs;					\
78 #define for_nexthops(fi) {						\  argument
79 	int nhsel; const struct fib_nh *nh = (fi)->fib_nh;		\
82 #define change_nexthops(fi) {						\  argument
84 	struct fib_nh *nexthop_nh = (struct fib_nh *)((fi)->fib_nh);	\
89 #define endfor_nexthops(fi) }  argument
206 	struct fib_info *fi = container_of(head, struct fib_info, rcu);  in free_fib_info_rcu()  local
208 	change_nexthops(fi) {  in free_fib_info_rcu()
215 	} endfor_nexthops(fi);  in free_fib_info_rcu()
217 	if (fi->fib_metrics != (u32 *) dst_default_metrics)  in free_fib_info_rcu()
218 		kfree(fi->fib_metrics);  in free_fib_info_rcu()
219 	kfree(fi);  in free_fib_info_rcu()
222 void free_fib_info(struct fib_info *fi)  in free_fib_info()  argument
224 	if (fi->fib_dead == 0) {  in free_fib_info()
225 		pr_warn("Freeing alive fib_info %p\n", fi);  in free_fib_info()
230 	change_nexthops(fi) {  in free_fib_info()
232 			fi->fib_net->ipv4.fib_num_tclassid_users--;  in free_fib_info()
233 	} endfor_nexthops(fi);  in free_fib_info()
235 	call_rcu(&fi->rcu, free_fib_info_rcu);  in free_fib_info()
238 void fib_release_info(struct fib_info *fi)  in fib_release_info()  argument
241 	if (fi && --fi->fib_treeref == 0) {  in fib_release_info()
242 		hlist_del(&fi->fib_hash);  in fib_release_info()
243 		if (fi->fib_prefsrc)  in fib_release_info()
244 			hlist_del(&fi->fib_lhash);  in fib_release_info()
245 		change_nexthops(fi) {  in fib_release_info()
249 		} endfor_nexthops(fi)  in fib_release_info()
250 		fi->fib_dead = 1;  in fib_release_info()
251 		fib_info_put(fi);  in fib_release_info()
256 static inline int nh_comp(const struct fib_info *fi, const struct fib_info *ofi)  in nh_comp()  argument
260 	for_nexthops(fi) {  in nh_comp()
274 	} endfor_nexthops(fi);  in nh_comp()
287 static inline unsigned int fib_info_hashfn(const struct fib_info *fi)  in fib_info_hashfn()  argument
290 	unsigned int val = fi->fib_nhs;  in fib_info_hashfn()
292 	val ^= (fi->fib_protocol << 8) | fi->fib_scope;  in fib_info_hashfn()
293 	val ^= (__force u32)fi->fib_prefsrc;  in fib_info_hashfn()
294 	val ^= fi->fib_priority;  in fib_info_hashfn()
295 	for_nexthops(fi) {  in fib_info_hashfn()
297 	} endfor_nexthops(fi)  in fib_info_hashfn()
305 	struct fib_info *fi;  in fib_find_info()  local
311 	hlist_for_each_entry(fi, head, fib_hash) {  in fib_find_info()
312 		if (!net_eq(fi->fib_net, nfi->fib_net))  in fib_find_info()
314 		if (fi->fib_nhs != nfi->fib_nhs)  in fib_find_info()
316 		if (nfi->fib_protocol == fi->fib_protocol &&  in fib_find_info()
317 		    nfi->fib_scope == fi->fib_scope &&  in fib_find_info()
318 		    nfi->fib_prefsrc == fi->fib_prefsrc &&  in fib_find_info()
319 		    nfi->fib_priority == fi->fib_priority &&  in fib_find_info()
320 		    nfi->fib_type == fi->fib_type &&  in fib_find_info()
321 		    memcmp(nfi->fib_metrics, fi->fib_metrics,  in fib_find_info()
323 		    !((nfi->fib_flags ^ fi->fib_flags) & ~RTNH_COMPARE_MASK) &&  in fib_find_info()
324 		    (nfi->fib_nhs == 0 || nh_comp(fi, nfi) == 0))  in fib_find_info()
325 			return fi;  in fib_find_info()
358 static inline size_t fib_nlmsg_size(struct fib_info *fi)  in fib_nlmsg_size()  argument
370 	if (fi->fib_nhs) {  in fib_nlmsg_size()
381 		for_nexthops(fi) {  in fib_nlmsg_size()
389 		} endfor_nexthops(fi);  in fib_nlmsg_size()
392 		payload += nla_total_size((fi->fib_nhs * nhsize) +  in fib_nlmsg_size()
429 static int fib_detect_death(struct fib_info *fi, int order,  in fib_detect_death()  argument
436 	n = neigh_lookup(&arp_tbl, &fi->fib_nh[0].nh_gw, fi->fib_dev);  in fib_detect_death()
449 		*last_resort = fi;  in fib_detect_death()
470 static int fib_get_nhs(struct fib_info *fi, struct rtnexthop *rtnh,  in fib_get_nhs()  argument
476 	change_nexthops(fi) {  in fib_get_nhs()
497 				fi->fib_net->ipv4.fib_num_tclassid_users++;  in fib_get_nhs()
523 	} endfor_nexthops(fi);  in fib_get_nhs()
534 static void fib_rebalance(struct fib_info *fi)  in fib_rebalance()  argument
540 	if (fi->fib_nhs < 2)  in fib_rebalance()
544 	for_nexthops(fi) {  in fib_rebalance()
556 	} endfor_nexthops(fi);  in fib_rebalance()
559 	change_nexthops(fi) {  in fib_rebalance()
577 	} endfor_nexthops(fi);  in fib_rebalance()
583 static inline void fib_add_weight(struct fib_info *fi,  in fib_add_weight()  argument
586 	fi->fib_weight += nh->nh_weight;  in fib_add_weight()
591 #define fib_rebalance(fi) do { } while (0)  argument
592 #define fib_add_weight(fi, nh) do { } while (0)  argument
620 int fib_nh_match(struct fib_config *cfg, struct fib_info *fi)  in fib_nh_match()  argument
628 	if (cfg->fc_priority && cfg->fc_priority != fi->fib_priority)  in fib_nh_match()
635 					    fi->fib_nh, cfg))  in fib_nh_match()
638 		if ((!cfg->fc_oif || cfg->fc_oif == fi->fib_nh->nh_oif) &&  in fib_nh_match()
639 		    (!cfg->fc_gw  || cfg->fc_gw == fi->fib_nh->nh_gw))  in fib_nh_match()
651 	for_nexthops(fi) {  in fib_nh_match()
675 	} endfor_nexthops(fi);  in fib_nh_match()
724 static int fib_check_nh(struct fib_config *cfg, struct fib_info *fi,  in fib_check_nh()  argument
875 		struct fib_info *fi;  in fib_info_hash_move()  local
877 		hlist_for_each_entry_safe(fi, n, head, fib_hash) {  in fib_info_hash_move()
881 			new_hash = fib_info_hashfn(fi);  in fib_info_hash_move()
883 			hlist_add_head(&fi->fib_hash, dest);  in fib_info_hash_move()
891 		struct fib_info *fi;  in fib_info_hash_move()  local
893 		hlist_for_each_entry_safe(fi, n, lhead, fib_lhash) {  in fib_info_hash_move()
897 			new_hash = fib_laddr_hashfn(fi->fib_prefsrc);  in fib_info_hash_move()
899 			hlist_add_head(&fi->fib_lhash, ldest);  in fib_info_hash_move()
946 fib_convert_metrics(struct fib_info *fi, const struct fib_config *cfg)  in fib_convert_metrics()  argument
982 		fi->fib_metrics[type - 1] = val;  in fib_convert_metrics()
986 		fi->fib_metrics[RTAX_FEATURES - 1] |= DST_FEATURE_ECN_CA;  in fib_convert_metrics()
994 	struct fib_info *fi = NULL;  in fib_create_info()  local
1036 	fi = kzalloc(sizeof(*fi)+nhs*sizeof(struct fib_nh), GFP_KERNEL);  in fib_create_info()
1037 	if (!fi)  in fib_create_info()
1041 		fi->fib_metrics = kzalloc(sizeof(u32) * RTAX_MAX, GFP_KERNEL);  in fib_create_info()
1042 		if (!fi->fib_metrics)  in fib_create_info()
1045 		fi->fib_metrics = (u32 *) dst_default_metrics;  in fib_create_info()
1047 	fi->fib_net = net;  in fib_create_info()
1048 	fi->fib_protocol = cfg->fc_protocol;  in fib_create_info()
1049 	fi->fib_scope = cfg->fc_scope;  in fib_create_info()
1050 	fi->fib_flags = cfg->fc_flags;  in fib_create_info()
1051 	fi->fib_priority = cfg->fc_priority;  in fib_create_info()
1052 	fi->fib_prefsrc = cfg->fc_prefsrc;  in fib_create_info()
1053 	fi->fib_type = cfg->fc_type;  in fib_create_info()
1055 	fi->fib_nhs = nhs;  in fib_create_info()
1056 	change_nexthops(fi) {  in fib_create_info()
1057 		nexthop_nh->nh_parent = fi;  in fib_create_info()
1061 	} endfor_nexthops(fi)  in fib_create_info()
1063 	err = fib_convert_metrics(fi, cfg);  in fib_create_info()
1069 		err = fib_get_nhs(fi, cfg->fc_mp, cfg->fc_mp_len, cfg);  in fib_create_info()
1072 		if (cfg->fc_oif && fi->fib_nh->nh_oif != cfg->fc_oif)  in fib_create_info()
1074 		if (cfg->fc_gw && fi->fib_nh->nh_gw != cfg->fc_gw)  in fib_create_info()
1077 		if (cfg->fc_flow && fi->fib_nh->nh_tclassid != cfg->fc_flow)  in fib_create_info()
1084 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1108 			fi->fib_net->ipv4.fib_num_tclassid_users++;  in fib_create_info()
1136 		struct fib_nh *nh = fi->fib_nh;  in fib_create_info()
1142 		nh->nh_dev = dev_get_by_index(net, fi->fib_nh->nh_oif);  in fib_create_info()
1149 		change_nexthops(fi) {  in fib_create_info()
1150 			err = fib_check_nh(cfg, fi, nexthop_nh);  in fib_create_info()
1155 		} endfor_nexthops(fi)  in fib_create_info()
1156 		if (linkdown == fi->fib_nhs)  in fib_create_info()
1157 			fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_create_info()
1160 	if (fi->fib_prefsrc && !fib_valid_prefsrc(cfg, fi->fib_prefsrc))  in fib_create_info()
1163 	change_nexthops(fi) {  in fib_create_info()
1165 		fib_add_weight(fi, nexthop_nh);  in fib_create_info()
1166 	} endfor_nexthops(fi)  in fib_create_info()
1168 	fib_rebalance(fi);  in fib_create_info()
1171 	ofi = fib_find_info(fi);  in fib_create_info()
1173 		fi->fib_dead = 1;  in fib_create_info()
1174 		free_fib_info(fi);  in fib_create_info()
1179 	fi->fib_treeref++;  in fib_create_info()
1180 	atomic_inc(&fi->fib_clntref);  in fib_create_info()
1182 	hlist_add_head(&fi->fib_hash,  in fib_create_info()
1183 		       &fib_info_hash[fib_info_hashfn(fi)]);  in fib_create_info()
1184 	if (fi->fib_prefsrc) {  in fib_create_info()
1187 		head = &fib_info_laddrhash[fib_laddr_hashfn(fi->fib_prefsrc)];  in fib_create_info()
1188 		hlist_add_head(&fi->fib_lhash, head);  in fib_create_info()
1190 	change_nexthops(fi) {  in fib_create_info()
1199 	} endfor_nexthops(fi)  in fib_create_info()
1201 	return fi;  in fib_create_info()
1207 	if (fi) {  in fib_create_info()
1208 		fi->fib_dead = 1;  in fib_create_info()
1209 		free_fib_info(fi);  in fib_create_info()
1217 		  struct fib_info *fi, unsigned int flags)  in fib_dump_info()  argument
1238 	rtm->rtm_flags = fi->fib_flags;  in fib_dump_info()
1239 	rtm->rtm_scope = fi->fib_scope;  in fib_dump_info()
1240 	rtm->rtm_protocol = fi->fib_protocol;  in fib_dump_info()
1245 	if (fi->fib_priority &&  in fib_dump_info()
1246 	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority))  in fib_dump_info()
1248 	if (rtnetlink_put_metrics(skb, fi->fib_metrics) < 0)  in fib_dump_info()
1251 	if (fi->fib_prefsrc &&  in fib_dump_info()
1252 	    nla_put_in_addr(skb, RTA_PREFSRC, fi->fib_prefsrc))  in fib_dump_info()
1254 	if (fi->fib_nhs == 1) {  in fib_dump_info()
1257 		if (fi->fib_nh->nh_gw &&  in fib_dump_info()
1258 		    nla_put_in_addr(skb, RTA_GATEWAY, fi->fib_nh->nh_gw))  in fib_dump_info()
1260 		if (fi->fib_nh->nh_oif &&  in fib_dump_info()
1261 		    nla_put_u32(skb, RTA_OIF, fi->fib_nh->nh_oif))  in fib_dump_info()
1263 		if (fi->fib_nh->nh_flags & RTNH_F_LINKDOWN) {  in fib_dump_info()
1264 			in_dev = __in_dev_get_rtnl(fi->fib_nh->nh_dev);  in fib_dump_info()
1270 		if (fi->fib_nh[0].nh_tclassid &&  in fib_dump_info()
1271 		    nla_put_u32(skb, RTA_FLOW, fi->fib_nh[0].nh_tclassid))  in fib_dump_info()
1274 		if (fi->fib_nh->nh_lwtstate)  in fib_dump_info()
1275 			lwtunnel_fill_encap(skb, fi->fib_nh->nh_lwtstate);  in fib_dump_info()
1278 	if (fi->fib_nhs > 1) {  in fib_dump_info()
1286 		for_nexthops(fi) {  in fib_dump_info()
1315 		} endfor_nexthops(fi);  in fib_dump_info()
1339 	struct fib_info *fi;  in fib_sync_down_addr()  local
1344 	hlist_for_each_entry(fi, head, fib_lhash) {  in fib_sync_down_addr()
1345 		if (!net_eq(fi->fib_net, net))  in fib_sync_down_addr()
1347 		if (fi->fib_prefsrc == local) {  in fib_sync_down_addr()
1348 			fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_addr()
1374 		struct fib_info *fi = nh->nh_parent;  in fib_sync_down_dev()  local
1377 		BUG_ON(!fi->fib_nhs);  in fib_sync_down_dev()
1378 		if (nh->nh_dev != dev || fi == prev_fi)  in fib_sync_down_dev()
1380 		prev_fi = fi;  in fib_sync_down_dev()
1382 		change_nexthops(fi) {  in fib_sync_down_dev()
1401 				dead = fi->fib_nhs;  in fib_sync_down_dev()
1405 		} endfor_nexthops(fi)  in fib_sync_down_dev()
1406 		if (dead == fi->fib_nhs) {  in fib_sync_down_dev()
1410 				fi->fib_flags |= RTNH_F_DEAD;  in fib_sync_down_dev()
1413 				fi->fib_flags |= RTNH_F_LINKDOWN;  in fib_sync_down_dev()
1419 		fib_rebalance(fi);  in fib_sync_down_dev()
1428 	struct fib_info *fi = NULL, *last_resort = NULL;  in fib_select_default()  local
1434 	u32 last_prio = res->fi->fib_priority;  in fib_select_default()
1466 		if (!fi) {  in fib_select_default()
1467 			if (next_fi != res->fi)  in fib_select_default()
1470 		} else if (!fib_detect_death(fi, order, &last_resort,  in fib_select_default()
1472 			fib_result_assign(res, fi);  in fib_select_default()
1476 		fi = next_fi;  in fib_select_default()
1480 	if (order <= 0 || !fi) {  in fib_select_default()
1486 	if (!fib_detect_death(fi, order, &last_resort, &last_idx,  in fib_select_default()
1488 		fib_result_assign(res, fi);  in fib_select_default()
1528 		struct fib_info *fi = nh->nh_parent;  in fib_sync_up()  local
1531 		BUG_ON(!fi->fib_nhs);  in fib_sync_up()
1532 		if (nh->nh_dev != dev || fi == prev_fi)  in fib_sync_up()
1535 		prev_fi = fi;  in fib_sync_up()
1537 		change_nexthops(fi) {  in fib_sync_up()
1550 		} endfor_nexthops(fi)  in fib_sync_up()
1553 			fi->fib_flags &= ~nh_flags;  in fib_sync_up()
1557 		fib_rebalance(fi);  in fib_sync_up()
1567 	struct fib_info *fi = res->fi;  in fib_select_multipath()  local
1569 	for_nexthops(fi) {  in fib_select_multipath()
1575 	} endfor_nexthops(fi);  in fib_select_multipath()
1586 	if (res->fi->fib_nhs > 1 && fl4->flowi4_oif == 0) {  in fib_select_path()