85 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_set_flow_control()  local
89 		ic->i_flowctl = 1;  in rds_ib_set_flow_control()
92 		ic->i_flowctl = 0;  in rds_ib_set_flow_control()
111 rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr)  in rds_ib_tune_rnr()  argument
116 	ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);  in rds_ib_tune_rnr()
128 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_connect_complete()  local
157 		       ic->i_flowctl ? ", flow control" : "");  in rds_ib_cm_connect_complete()
164 	rds_ib_send_init_ring(ic);  in rds_ib_cm_connect_complete()
165 	rds_ib_recv_init_ring(ic);  in rds_ib_cm_connect_complete()
171 	rds_ib_tune_rnr(ic, &qp_attr);  in rds_ib_cm_connect_complete()
174 	err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);  in rds_ib_cm_connect_complete()
179 	err = rds_ib_update_ipaddr(ic->rds_ibdev, conn->c_laddr);  in rds_ib_cm_connect_complete()
208 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_fill_conn_param()  local
209 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;  in rds_ib_cm_fill_conn_param()
227 		dp->dp_ack_seq = rds_ib_piggyb_ack(ic);  in rds_ib_cm_fill_conn_param()
230 		if (ic->i_flowctl) {  in rds_ib_cm_fill_conn_param()
233 			credits = IB_GET_POST_CREDITS(atomic_read(&ic->i_credits));  in rds_ib_cm_fill_conn_param()
235 			atomic_sub(IB_SET_POST_CREDITS(credits), &ic->i_credits);  in rds_ib_cm_fill_conn_param()
252 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_qp_event_handler()  local
254 	rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event,  in rds_ib_qp_event_handler()
259 		rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);  in rds_ib_qp_event_handler()
277 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_setup_qp()  local
278 	struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_setup_qp()
294 	if (rds_ibdev->max_wrs < ic->i_send_ring.w_nr + 1)  in rds_ib_setup_qp()
295 		rds_ib_ring_resize(&ic->i_send_ring, rds_ibdev->max_wrs - 1);  in rds_ib_setup_qp()
296 	if (rds_ibdev->max_wrs < ic->i_recv_ring.w_nr + 1)  in rds_ib_setup_qp()
297 		rds_ib_ring_resize(&ic->i_recv_ring, rds_ibdev->max_wrs - 1);  in rds_ib_setup_qp()
300 	ic->i_pd = rds_ibdev->pd;  in rds_ib_setup_qp()
301 	ic->i_mr = rds_ibdev->mr;  in rds_ib_setup_qp()
303 	ic->i_send_cq = ib_create_cq(dev, rds_ib_send_cq_comp_handler,  in rds_ib_setup_qp()
305 				     ic->i_send_ring.w_nr + 1, 0);  in rds_ib_setup_qp()
306 	if (IS_ERR(ic->i_send_cq)) {  in rds_ib_setup_qp()
307 		ret = PTR_ERR(ic->i_send_cq);  in rds_ib_setup_qp()
308 		ic->i_send_cq = NULL;  in rds_ib_setup_qp()
313 	ic->i_recv_cq = ib_create_cq(dev, rds_ib_recv_cq_comp_handler,  in rds_ib_setup_qp()
315 				     ic->i_recv_ring.w_nr, 0);  in rds_ib_setup_qp()
316 	if (IS_ERR(ic->i_recv_cq)) {  in rds_ib_setup_qp()
317 		ret = PTR_ERR(ic->i_recv_cq);  in rds_ib_setup_qp()
318 		ic->i_recv_cq = NULL;  in rds_ib_setup_qp()
323 	ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);  in rds_ib_setup_qp()
329 	ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);  in rds_ib_setup_qp()
340 	attr.cap.max_send_wr = ic->i_send_ring.w_nr + 1;  in rds_ib_setup_qp()
341 	attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;  in rds_ib_setup_qp()
346 	attr.send_cq = ic->i_send_cq;  in rds_ib_setup_qp()
347 	attr.recv_cq = ic->i_recv_cq;  in rds_ib_setup_qp()
353 	ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);  in rds_ib_setup_qp()
359 	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,  in rds_ib_setup_qp()
360 					   ic->i_send_ring.w_nr *  in rds_ib_setup_qp()
362 					   &ic->i_send_hdrs_dma, GFP_KERNEL);  in rds_ib_setup_qp()
363 	if (!ic->i_send_hdrs) {  in rds_ib_setup_qp()
369 	ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,  in rds_ib_setup_qp()
370 					   ic->i_recv_ring.w_nr *  in rds_ib_setup_qp()
372 					   &ic->i_recv_hdrs_dma, GFP_KERNEL);  in rds_ib_setup_qp()
373 	if (!ic->i_recv_hdrs) {  in rds_ib_setup_qp()
379 	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),  in rds_ib_setup_qp()
380 				       &ic->i_ack_dma, GFP_KERNEL);  in rds_ib_setup_qp()
381 	if (!ic->i_ack) {  in rds_ib_setup_qp()
387 	ic->i_sends = vzalloc_node(ic->i_send_ring.w_nr * sizeof(struct rds_ib_send_work),  in rds_ib_setup_qp()
389 	if (!ic->i_sends) {  in rds_ib_setup_qp()
395 	ic->i_recvs = vzalloc_node(ic->i_recv_ring.w_nr * sizeof(struct rds_ib_recv_work),  in rds_ib_setup_qp()
397 	if (!ic->i_recvs) {  in rds_ib_setup_qp()
403 	rds_ib_recv_init_ack(ic);  in rds_ib_setup_qp()
405 	rdsdebug("conn %p pd %p mr %p cq %p %p\n", conn, ic->i_pd, ic->i_mr,  in rds_ib_setup_qp()
406 		 ic->i_send_cq, ic->i_recv_cq);  in rds_ib_setup_qp()
462 	struct rds_ib_connection *ic = NULL;  in rds_ib_cm_handle_connect()  local
507 	ic = conn->c_transport_data;  in rds_ib_cm_handle_connect()
518 	BUG_ON(ic->i_cm_id);  in rds_ib_cm_handle_connect()
520 	ic->i_cm_id = cm_id;  in rds_ib_cm_handle_connect()
554 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_initiate_connect()  local
562 	ic->i_flowctl = rds_ib_sysctl_flow_control;	/* advertise flow control */  in rds_ib_cm_initiate_connect()
581 		if (ic->i_cm_id == cm_id)  in rds_ib_cm_initiate_connect()
589 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_conn_connect()  local
595 	ic->i_cm_id = rdma_create_id(rds_rdma_cm_event_handler, conn,  in rds_ib_conn_connect()
597 	if (IS_ERR(ic->i_cm_id)) {  in rds_ib_conn_connect()
598 		ret = PTR_ERR(ic->i_cm_id);  in rds_ib_conn_connect()
599 		ic->i_cm_id = NULL;  in rds_ib_conn_connect()
604 	rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);  in rds_ib_conn_connect()
614 	ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,  in rds_ib_conn_connect()
618 		rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,  in rds_ib_conn_connect()
620 		rdma_destroy_id(ic->i_cm_id);  in rds_ib_conn_connect()
621 		ic->i_cm_id = NULL;  in rds_ib_conn_connect()
635 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_conn_shutdown()  local
638 	rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,  in rds_ib_conn_shutdown()
639 		 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,  in rds_ib_conn_shutdown()
640 		 ic->i_cm_id ? ic->i_cm_id->qp : NULL);  in rds_ib_conn_shutdown()
642 	if (ic->i_cm_id) {  in rds_ib_conn_shutdown()
643 		struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_conn_shutdown()
645 		rdsdebug("disconnecting cm %p\n", ic->i_cm_id);  in rds_ib_conn_shutdown()
646 		err = rdma_disconnect(ic->i_cm_id);  in rds_ib_conn_shutdown()
652 				ic->i_cm_id, err);  in rds_ib_conn_shutdown()
665 			   rds_ib_ring_empty(&ic->i_recv_ring) &&  in rds_ib_conn_shutdown()
666 			   (atomic_read(&ic->i_signaled_sends) == 0));  in rds_ib_conn_shutdown()
667 		tasklet_kill(&ic->i_recv_tasklet);  in rds_ib_conn_shutdown()
669 		if (ic->i_send_hdrs)  in rds_ib_conn_shutdown()
671 					   ic->i_send_ring.w_nr *  in rds_ib_conn_shutdown()
673 					   ic->i_send_hdrs,  in rds_ib_conn_shutdown()
674 					   ic->i_send_hdrs_dma);  in rds_ib_conn_shutdown()
676 		if (ic->i_recv_hdrs)  in rds_ib_conn_shutdown()
678 					   ic->i_recv_ring.w_nr *  in rds_ib_conn_shutdown()
680 					   ic->i_recv_hdrs,  in rds_ib_conn_shutdown()
681 					   ic->i_recv_hdrs_dma);  in rds_ib_conn_shutdown()
683 		if (ic->i_ack)  in rds_ib_conn_shutdown()
685 					     ic->i_ack, ic->i_ack_dma);  in rds_ib_conn_shutdown()
687 		if (ic->i_sends)  in rds_ib_conn_shutdown()
688 			rds_ib_send_clear_ring(ic);  in rds_ib_conn_shutdown()
689 		if (ic->i_recvs)  in rds_ib_conn_shutdown()
690 			rds_ib_recv_clear_ring(ic);  in rds_ib_conn_shutdown()
692 		if (ic->i_cm_id->qp)  in rds_ib_conn_shutdown()
693 			rdma_destroy_qp(ic->i_cm_id);  in rds_ib_conn_shutdown()
694 		if (ic->i_send_cq)  in rds_ib_conn_shutdown()
695 			ib_destroy_cq(ic->i_send_cq);  in rds_ib_conn_shutdown()
696 		if (ic->i_recv_cq)  in rds_ib_conn_shutdown()
697 			ib_destroy_cq(ic->i_recv_cq);  in rds_ib_conn_shutdown()
698 		rdma_destroy_id(ic->i_cm_id);  in rds_ib_conn_shutdown()
703 		if (ic->rds_ibdev)  in rds_ib_conn_shutdown()
704 			rds_ib_remove_conn(ic->rds_ibdev, conn);  in rds_ib_conn_shutdown()
706 		ic->i_cm_id = NULL;  in rds_ib_conn_shutdown()
707 		ic->i_pd = NULL;  in rds_ib_conn_shutdown()
708 		ic->i_mr = NULL;  in rds_ib_conn_shutdown()
709 		ic->i_send_cq = NULL;  in rds_ib_conn_shutdown()
710 		ic->i_recv_cq = NULL;  in rds_ib_conn_shutdown()
711 		ic->i_send_hdrs = NULL;  in rds_ib_conn_shutdown()
712 		ic->i_recv_hdrs = NULL;  in rds_ib_conn_shutdown()
713 		ic->i_ack = NULL;  in rds_ib_conn_shutdown()
715 	BUG_ON(ic->rds_ibdev);  in rds_ib_conn_shutdown()
718 	if (ic->i_data_op) {  in rds_ib_conn_shutdown()
721 		rm = container_of(ic->i_data_op, struct rds_message, data);  in rds_ib_conn_shutdown()
723 		ic->i_data_op = NULL;  in rds_ib_conn_shutdown()
727 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_ib_conn_shutdown()
729 	atomic64_set(&ic->i_ack_next, 0);  in rds_ib_conn_shutdown()
731 	ic->i_ack_next = 0;  in rds_ib_conn_shutdown()
733 	ic->i_ack_recv = 0;  in rds_ib_conn_shutdown()
736 	ic->i_flowctl = 0;  in rds_ib_conn_shutdown()
737 	atomic_set(&ic->i_credits, 0);  in rds_ib_conn_shutdown()
739 	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);  in rds_ib_conn_shutdown()
740 	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);  in rds_ib_conn_shutdown()
742 	if (ic->i_ibinc) {  in rds_ib_conn_shutdown()
743 		rds_inc_put(&ic->i_ibinc->ii_inc);  in rds_ib_conn_shutdown()
744 		ic->i_ibinc = NULL;  in rds_ib_conn_shutdown()
747 	vfree(ic->i_sends);  in rds_ib_conn_shutdown()
748 	ic->i_sends = NULL;  in rds_ib_conn_shutdown()
749 	vfree(ic->i_recvs);  in rds_ib_conn_shutdown()
750 	ic->i_recvs = NULL;  in rds_ib_conn_shutdown()
755 	struct rds_ib_connection *ic;  in rds_ib_conn_alloc()  local
760 	ic = kzalloc(sizeof(struct rds_ib_connection), gfp);  in rds_ib_conn_alloc()
761 	if (!ic)  in rds_ib_conn_alloc()
764 	ret = rds_ib_recv_alloc_caches(ic);  in rds_ib_conn_alloc()
766 		kfree(ic);  in rds_ib_conn_alloc()
770 	INIT_LIST_HEAD(&ic->ib_node);  in rds_ib_conn_alloc()
771 	tasklet_init(&ic->i_recv_tasklet, rds_ib_recv_tasklet_fn,  in rds_ib_conn_alloc()
772 		     (unsigned long) ic);  in rds_ib_conn_alloc()
773 	mutex_init(&ic->i_recv_mutex);  in rds_ib_conn_alloc()
775 	spin_lock_init(&ic->i_ack_lock);  in rds_ib_conn_alloc()
777 	atomic_set(&ic->i_signaled_sends, 0);  in rds_ib_conn_alloc()
783 	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);  in rds_ib_conn_alloc()
784 	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);  in rds_ib_conn_alloc()
786 	ic->conn = conn;  in rds_ib_conn_alloc()
787 	conn->c_transport_data = ic;  in rds_ib_conn_alloc()
790 	list_add_tail(&ic->ib_node, &ib_nodev_conns);  in rds_ib_conn_alloc()
803 	struct rds_ib_connection *ic = arg;  in rds_ib_conn_free()  local
806 	rdsdebug("ic %p\n", ic);  in rds_ib_conn_free()
813 	lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;  in rds_ib_conn_free()
816 	list_del(&ic->ib_node);  in rds_ib_conn_free()
819 	rds_ib_recv_free_caches(ic);  in rds_ib_conn_free()
821 	kfree(ic);  in rds_ib_conn_free()