442 	struct ipoib_neigh *neigh, *tn;  in path_rec_completion()  local
477 		list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {  in path_rec_completion()
478 			if (neigh->ah) {  in path_rec_completion()
479 				WARN_ON(neigh->ah != old_ah);  in path_rec_completion()
487 				ipoib_put_ah(neigh->ah);  in path_rec_completion()
490 			neigh->ah = path->ah;  in path_rec_completion()
492 			if (ipoib_cm_enabled(dev, neigh->daddr)) {  in path_rec_completion()
493 				if (!ipoib_cm_get(neigh))  in path_rec_completion()
494 					ipoib_cm_set(neigh, ipoib_cm_create_tx(dev,  in path_rec_completion()
496 									       neigh));  in path_rec_completion()
497 				if (!ipoib_cm_get(neigh)) {  in path_rec_completion()
498 					ipoib_neigh_free(neigh);  in path_rec_completion()
503 			while ((skb = __skb_dequeue(&neigh->queue)))  in path_rec_completion()
591 	struct ipoib_neigh *neigh;  in neigh_add_path()  local
595 	neigh = ipoib_neigh_alloc(daddr, dev);  in neigh_add_path()
596 	if (!neigh) {  in neigh_add_path()
612 	list_add_tail(&neigh->list, &path->neigh_list);  in neigh_add_path()
616 		neigh->ah = path->ah;  in neigh_add_path()
618 		if (ipoib_cm_enabled(dev, neigh->daddr)) {  in neigh_add_path()
619 			if (!ipoib_cm_get(neigh))  in neigh_add_path()
620 				ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));  in neigh_add_path()
621 			if (!ipoib_cm_get(neigh)) {  in neigh_add_path()
622 				ipoib_neigh_free(neigh);  in neigh_add_path()
625 			if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE)  in neigh_add_path()
626 				__skb_queue_tail(&neigh->queue, skb);  in neigh_add_path()
629 					   skb_queue_len(&neigh->queue));  in neigh_add_path()
635 			ipoib_neigh_put(neigh);  in neigh_add_path()
639 		neigh->ah  = NULL;  in neigh_add_path()
643 		if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE)  in neigh_add_path()
644 			__skb_queue_tail(&neigh->queue, skb);  in neigh_add_path()
650 	ipoib_neigh_put(neigh);  in neigh_add_path()
654 	ipoib_neigh_free(neigh);  in neigh_add_path()
660 	ipoib_neigh_put(neigh);  in neigh_add_path()
725 	struct ipoib_neigh *neigh;  in ipoib_start_xmit()  local
748 		neigh = ipoib_neigh_get(dev, cb->hwaddr);  in ipoib_start_xmit()
749 		if (likely(neigh))  in ipoib_start_xmit()
760 		neigh = ipoib_neigh_get(dev, cb->hwaddr);  in ipoib_start_xmit()
761 		if (unlikely(!neigh)) {  in ipoib_start_xmit()
780 	if (ipoib_cm_get(neigh)) {  in ipoib_start_xmit()
781 		if (ipoib_cm_up(neigh)) {  in ipoib_start_xmit()
782 			ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));  in ipoib_start_xmit()
785 	} else if (neigh->ah) {  in ipoib_start_xmit()
786 		ipoib_send(dev, skb, neigh->ah, IPOIB_QPN(cb->hwaddr));  in ipoib_start_xmit()
790 	if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {  in ipoib_start_xmit()
792 		__skb_queue_tail(&neigh->queue, skb);  in ipoib_start_xmit()
800 	ipoib_neigh_put(neigh);  in ipoib_start_xmit()
885 	struct ipoib_neigh *neigh = NULL;  in ipoib_neigh_get()  local
896 	for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);  in ipoib_neigh_get()
897 	     neigh != NULL;  in ipoib_neigh_get()
898 	     neigh = rcu_dereference_bh(neigh->hnext)) {  in ipoib_neigh_get()
899 		if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {  in ipoib_neigh_get()
901 			if (!atomic_inc_not_zero(&neigh->refcnt)) {  in ipoib_neigh_get()
903 				neigh = NULL;  in ipoib_neigh_get()
906 			neigh->alive = jiffies;  in ipoib_neigh_get()
913 	return neigh;  in ipoib_neigh_get()
944 		struct ipoib_neigh *neigh;  in __ipoib_reap_neigh()  local
947 		while ((neigh = rcu_dereference_protected(*np,  in __ipoib_reap_neigh()
950 			if (time_after(neigh_obsolete, neigh->alive)) {  in __ipoib_reap_neigh()
952 						   rcu_dereference_protected(neigh->hnext,  in __ipoib_reap_neigh()
955 				list_del(&neigh->list);  in __ipoib_reap_neigh()
956 				call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in __ipoib_reap_neigh()
958 				np = &neigh->hnext;  in __ipoib_reap_neigh()
984 	struct ipoib_neigh *neigh;  in ipoib_neigh_ctor()  local
986 	neigh = kzalloc(sizeof *neigh, GFP_ATOMIC);  in ipoib_neigh_ctor()
987 	if (!neigh)  in ipoib_neigh_ctor()
990 	neigh->dev = dev;  in ipoib_neigh_ctor()
991 	memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));  in ipoib_neigh_ctor()
992 	skb_queue_head_init(&neigh->queue);  in ipoib_neigh_ctor()
993 	INIT_LIST_HEAD(&neigh->list);  in ipoib_neigh_ctor()
994 	ipoib_cm_set(neigh, NULL);  in ipoib_neigh_ctor()
996 	atomic_set(&neigh->refcnt, 1);  in ipoib_neigh_ctor()
998 	return neigh;  in ipoib_neigh_ctor()
1007 	struct ipoib_neigh *neigh;  in ipoib_neigh_alloc()  local
1013 		neigh = NULL;  in ipoib_neigh_alloc()
1021 	for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],  in ipoib_neigh_alloc()
1023 	     neigh != NULL;  in ipoib_neigh_alloc()
1024 	     neigh = rcu_dereference_protected(neigh->hnext,  in ipoib_neigh_alloc()
1026 		if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {  in ipoib_neigh_alloc()
1028 			if (!atomic_inc_not_zero(&neigh->refcnt)) {  in ipoib_neigh_alloc()
1030 				neigh = NULL;  in ipoib_neigh_alloc()
1033 			neigh->alive = jiffies;  in ipoib_neigh_alloc()
1038 	neigh = ipoib_neigh_ctor(daddr, dev);  in ipoib_neigh_alloc()
1039 	if (!neigh)  in ipoib_neigh_alloc()
1043 	atomic_inc(&neigh->refcnt);  in ipoib_neigh_alloc()
1044 	neigh->alive = jiffies;  in ipoib_neigh_alloc()
1046 	rcu_assign_pointer(neigh->hnext,  in ipoib_neigh_alloc()
1049 	rcu_assign_pointer(htbl->buckets[hash_val], neigh);  in ipoib_neigh_alloc()
1054 	return neigh;  in ipoib_neigh_alloc()
1057 void ipoib_neigh_dtor(struct ipoib_neigh *neigh)  in ipoib_neigh_dtor()  argument
1060 	struct net_device *dev = neigh->dev;  in ipoib_neigh_dtor()
1063 	if (neigh->ah)  in ipoib_neigh_dtor()
1064 		ipoib_put_ah(neigh->ah);  in ipoib_neigh_dtor()
1065 	while ((skb = __skb_dequeue(&neigh->queue))) {  in ipoib_neigh_dtor()
1069 	if (ipoib_cm_get(neigh))  in ipoib_neigh_dtor()
1070 		ipoib_cm_destroy_tx(ipoib_cm_get(neigh));  in ipoib_neigh_dtor()
1073 		  IPOIB_QPN(neigh->daddr),  in ipoib_neigh_dtor()
1074 		  neigh->daddr + 4);  in ipoib_neigh_dtor()
1075 	kfree(neigh);  in ipoib_neigh_dtor()
1085 	struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);  in ipoib_neigh_reclaim()  local
1087 	ipoib_neigh_put(neigh);  in ipoib_neigh_reclaim()
1090 void ipoib_neigh_free(struct ipoib_neigh *neigh)  in ipoib_neigh_free()  argument
1092 	struct net_device *dev = neigh->dev;  in ipoib_neigh_free()
1105 	hash_val = ipoib_addr_hash(htbl, neigh->daddr);  in ipoib_neigh_free()
1112 		if (n == neigh) {  in ipoib_neigh_free()
1115 					   rcu_dereference_protected(neigh->hnext,  in ipoib_neigh_free()
1118 			list_del(&neigh->list);  in ipoib_neigh_free()
1119 			call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in ipoib_neigh_free()
1192 		struct ipoib_neigh *neigh;  in ipoib_del_neighs_by_gid()  local
1195 		while ((neigh = rcu_dereference_protected(*np,  in ipoib_del_neighs_by_gid()
1198 			if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) {  in ipoib_del_neighs_by_gid()
1200 						   rcu_dereference_protected(neigh->hnext,  in ipoib_del_neighs_by_gid()
1203 				list_del(&neigh->list);  in ipoib_del_neighs_by_gid()
1204 				call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in ipoib_del_neighs_by_gid()
1206 				np = &neigh->hnext;  in ipoib_del_neighs_by_gid()
1236 		struct ipoib_neigh *neigh;  in ipoib_flush_neighs()  local
1239 		while ((neigh = rcu_dereference_protected(*np,  in ipoib_flush_neighs()
1242 					   rcu_dereference_protected(neigh->hnext,  in ipoib_flush_neighs()
1245 			list_del(&neigh->list);  in ipoib_flush_neighs()
1246 			call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in ipoib_flush_neighs()