670 	struct ipoib_neigh *neigh, *tn;  in path_rec_completion()  local
705 		list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {  in path_rec_completion()
706 			if (neigh->ah) {  in path_rec_completion()
707 				WARN_ON(neigh->ah != old_ah);  in path_rec_completion()
715 				ipoib_put_ah(neigh->ah);  in path_rec_completion()
718 			neigh->ah = path->ah;  in path_rec_completion()
720 			if (ipoib_cm_enabled(dev, neigh->daddr)) {  in path_rec_completion()
721 				if (!ipoib_cm_get(neigh))  in path_rec_completion()
722 					ipoib_cm_set(neigh, ipoib_cm_create_tx(dev,  in path_rec_completion()
724 									       neigh));  in path_rec_completion()
725 				if (!ipoib_cm_get(neigh)) {  in path_rec_completion()
726 					ipoib_neigh_free(neigh);  in path_rec_completion()
731 			while ((skb = __skb_dequeue(&neigh->queue)))  in path_rec_completion()
819 	struct ipoib_neigh *neigh;  in neigh_add_path()  local
823 	neigh = ipoib_neigh_alloc(daddr, dev);  in neigh_add_path()
824 	if (!neigh) {  in neigh_add_path()
840 	list_add_tail(&neigh->list, &path->neigh_list);  in neigh_add_path()
844 		neigh->ah = path->ah;  in neigh_add_path()
846 		if (ipoib_cm_enabled(dev, neigh->daddr)) {  in neigh_add_path()
847 			if (!ipoib_cm_get(neigh))  in neigh_add_path()
848 				ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));  in neigh_add_path()
849 			if (!ipoib_cm_get(neigh)) {  in neigh_add_path()
850 				ipoib_neigh_free(neigh);  in neigh_add_path()
853 			if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE)  in neigh_add_path()
854 				__skb_queue_tail(&neigh->queue, skb);  in neigh_add_path()
857 					   skb_queue_len(&neigh->queue));  in neigh_add_path()
863 			ipoib_neigh_put(neigh);  in neigh_add_path()
867 		neigh->ah  = NULL;  in neigh_add_path()
871 		if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE)  in neigh_add_path()
872 			__skb_queue_tail(&neigh->queue, skb);  in neigh_add_path()
878 	ipoib_neigh_put(neigh);  in neigh_add_path()
882 	ipoib_neigh_free(neigh);  in neigh_add_path()
888 	ipoib_neigh_put(neigh);  in neigh_add_path()
953 	struct ipoib_neigh *neigh;  in ipoib_start_xmit()  local
976 		neigh = ipoib_neigh_get(dev, cb->hwaddr);  in ipoib_start_xmit()
977 		if (likely(neigh))  in ipoib_start_xmit()
988 		neigh = ipoib_neigh_get(dev, cb->hwaddr);  in ipoib_start_xmit()
989 		if (unlikely(!neigh)) {  in ipoib_start_xmit()
1008 	if (ipoib_cm_get(neigh)) {  in ipoib_start_xmit()
1009 		if (ipoib_cm_up(neigh)) {  in ipoib_start_xmit()
1010 			ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));  in ipoib_start_xmit()
1013 	} else if (neigh->ah) {  in ipoib_start_xmit()
1014 		ipoib_send(dev, skb, neigh->ah, IPOIB_QPN(cb->hwaddr));  in ipoib_start_xmit()
1018 	if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {  in ipoib_start_xmit()
1020 		__skb_queue_tail(&neigh->queue, skb);  in ipoib_start_xmit()
1028 	ipoib_neigh_put(neigh);  in ipoib_start_xmit()
1113 	struct ipoib_neigh *neigh = NULL;  in ipoib_neigh_get()  local
1124 	for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);  in ipoib_neigh_get()
1125 	     neigh != NULL;  in ipoib_neigh_get()
1126 	     neigh = rcu_dereference_bh(neigh->hnext)) {  in ipoib_neigh_get()
1127 		if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {  in ipoib_neigh_get()
1129 			if (!atomic_inc_not_zero(&neigh->refcnt)) {  in ipoib_neigh_get()
1131 				neigh = NULL;  in ipoib_neigh_get()
1134 			neigh->alive = jiffies;  in ipoib_neigh_get()
1141 	return neigh;  in ipoib_neigh_get()
1175 		struct ipoib_neigh *neigh;  in __ipoib_reap_neigh()  local
1178 		while ((neigh = rcu_dereference_protected(*np,  in __ipoib_reap_neigh()
1181 			if (time_after(neigh_obsolete, neigh->alive)) {  in __ipoib_reap_neigh()
1182 				u8 *mgid = neigh->daddr + 4;  in __ipoib_reap_neigh()
1196 						   rcu_dereference_protected(neigh->hnext,  in __ipoib_reap_neigh()
1199 				list_del(&neigh->list);  in __ipoib_reap_neigh()
1200 				call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in __ipoib_reap_neigh()
1202 				np = &neigh->hnext;  in __ipoib_reap_neigh()
1232 	struct ipoib_neigh *neigh;  in ipoib_neigh_ctor()  local
1234 	neigh = kzalloc(sizeof *neigh, GFP_ATOMIC);  in ipoib_neigh_ctor()
1235 	if (!neigh)  in ipoib_neigh_ctor()
1238 	neigh->dev = dev;  in ipoib_neigh_ctor()
1239 	memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));  in ipoib_neigh_ctor()
1240 	skb_queue_head_init(&neigh->queue);  in ipoib_neigh_ctor()
1241 	INIT_LIST_HEAD(&neigh->list);  in ipoib_neigh_ctor()
1242 	ipoib_cm_set(neigh, NULL);  in ipoib_neigh_ctor()
1244 	atomic_set(&neigh->refcnt, 1);  in ipoib_neigh_ctor()
1246 	return neigh;  in ipoib_neigh_ctor()
1255 	struct ipoib_neigh *neigh;  in ipoib_neigh_alloc()  local
1261 		neigh = NULL;  in ipoib_neigh_alloc()
1269 	for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],  in ipoib_neigh_alloc()
1271 	     neigh != NULL;  in ipoib_neigh_alloc()
1272 	     neigh = rcu_dereference_protected(neigh->hnext,  in ipoib_neigh_alloc()
1274 		if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {  in ipoib_neigh_alloc()
1276 			if (!atomic_inc_not_zero(&neigh->refcnt)) {  in ipoib_neigh_alloc()
1278 				neigh = NULL;  in ipoib_neigh_alloc()
1281 			neigh->alive = jiffies;  in ipoib_neigh_alloc()
1286 	neigh = ipoib_neigh_ctor(daddr, dev);  in ipoib_neigh_alloc()
1287 	if (!neigh)  in ipoib_neigh_alloc()
1291 	atomic_inc(&neigh->refcnt);  in ipoib_neigh_alloc()
1292 	neigh->alive = jiffies;  in ipoib_neigh_alloc()
1294 	rcu_assign_pointer(neigh->hnext,  in ipoib_neigh_alloc()
1297 	rcu_assign_pointer(htbl->buckets[hash_val], neigh);  in ipoib_neigh_alloc()
1302 	return neigh;  in ipoib_neigh_alloc()
1305 void ipoib_neigh_dtor(struct ipoib_neigh *neigh)  in ipoib_neigh_dtor()  argument
1308 	struct net_device *dev = neigh->dev;  in ipoib_neigh_dtor()
1311 	if (neigh->ah)  in ipoib_neigh_dtor()
1312 		ipoib_put_ah(neigh->ah);  in ipoib_neigh_dtor()
1313 	while ((skb = __skb_dequeue(&neigh->queue))) {  in ipoib_neigh_dtor()
1317 	if (ipoib_cm_get(neigh))  in ipoib_neigh_dtor()
1318 		ipoib_cm_destroy_tx(ipoib_cm_get(neigh));  in ipoib_neigh_dtor()
1321 		  IPOIB_QPN(neigh->daddr),  in ipoib_neigh_dtor()
1322 		  neigh->daddr + 4);  in ipoib_neigh_dtor()
1323 	kfree(neigh);  in ipoib_neigh_dtor()
1333 	struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);  in ipoib_neigh_reclaim()  local
1335 	ipoib_neigh_put(neigh);  in ipoib_neigh_reclaim()
1338 void ipoib_neigh_free(struct ipoib_neigh *neigh)  in ipoib_neigh_free()  argument
1340 	struct net_device *dev = neigh->dev;  in ipoib_neigh_free()
1353 	hash_val = ipoib_addr_hash(htbl, neigh->daddr);  in ipoib_neigh_free()
1360 		if (n == neigh) {  in ipoib_neigh_free()
1363 					   rcu_dereference_protected(neigh->hnext,  in ipoib_neigh_free()
1366 			list_del(&neigh->list);  in ipoib_neigh_free()
1367 			call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in ipoib_neigh_free()
1440 		struct ipoib_neigh *neigh;  in ipoib_del_neighs_by_gid()  local
1443 		while ((neigh = rcu_dereference_protected(*np,  in ipoib_del_neighs_by_gid()
1446 			if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) {  in ipoib_del_neighs_by_gid()
1448 						   rcu_dereference_protected(neigh->hnext,  in ipoib_del_neighs_by_gid()
1451 				list_del(&neigh->list);  in ipoib_del_neighs_by_gid()
1452 				call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in ipoib_del_neighs_by_gid()
1454 				np = &neigh->hnext;  in ipoib_del_neighs_by_gid()
1484 		struct ipoib_neigh *neigh;  in ipoib_flush_neighs()  local
1487 		while ((neigh = rcu_dereference_protected(*np,  in ipoib_flush_neighs()
1490 					   rcu_dereference_protected(neigh->hnext,  in ipoib_flush_neighs()
1493 			list_del(&neigh->list);  in ipoib_flush_neighs()
1494 			call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in ipoib_flush_neighs()