55 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_set_flow_control()  local
59 		ic->i_flowctl = 1;  in rds_ib_set_flow_control()
62 		ic->i_flowctl = 0;  in rds_ib_set_flow_control()
81 rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr)  in rds_ib_tune_rnr()  argument
86 	ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);  in rds_ib_tune_rnr()
98 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_connect_complete()  local
127 		       ic->i_flowctl ? ", flow control" : "");  in rds_ib_cm_connect_complete()
134 	rds_ib_send_init_ring(ic);  in rds_ib_cm_connect_complete()
135 	rds_ib_recv_init_ring(ic);  in rds_ib_cm_connect_complete()
141 	rds_ib_tune_rnr(ic, &qp_attr);  in rds_ib_cm_connect_complete()
144 	err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);  in rds_ib_cm_connect_complete()
149 	err = rds_ib_update_ipaddr(ic->rds_ibdev, conn->c_laddr);  in rds_ib_cm_connect_complete()
178 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_fill_conn_param()  local
179 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;  in rds_ib_cm_fill_conn_param()
197 		dp->dp_ack_seq = rds_ib_piggyb_ack(ic);  in rds_ib_cm_fill_conn_param()
200 		if (ic->i_flowctl) {  in rds_ib_cm_fill_conn_param()
203 			credits = IB_GET_POST_CREDITS(atomic_read(&ic->i_credits));  in rds_ib_cm_fill_conn_param()
205 			atomic_sub(IB_SET_POST_CREDITS(credits), &ic->i_credits);  in rds_ib_cm_fill_conn_param()
230 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cq_comp_handler_recv()  local
236 	tasklet_schedule(&ic->i_recv_tasklet);  in rds_ib_cq_comp_handler_recv()
239 static void poll_cq(struct rds_ib_connection *ic, struct ib_cq *cq,  in poll_cq()  argument
255 				rds_ib_send_cqe_handler(ic, wc);  in poll_cq()
257 				rds_ib_recv_cqe_handler(ic, wc, ack_state);  in poll_cq()
264 	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;  in rds_ib_tasklet_fn_send()  local
265 	struct rds_connection *conn = ic->conn;  in rds_ib_tasklet_fn_send()
271 	poll_cq(ic, ic->i_send_cq, ic->i_send_wc, &state);  in rds_ib_tasklet_fn_send()
272 	ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);  in rds_ib_tasklet_fn_send()
273 	poll_cq(ic, ic->i_send_cq, ic->i_send_wc, &state);  in rds_ib_tasklet_fn_send()
278 		rds_send_xmit(ic->conn);  in rds_ib_tasklet_fn_send()
283 	struct rds_ib_connection *ic = (struct rds_ib_connection *)data;  in rds_ib_tasklet_fn_recv()  local
284 	struct rds_connection *conn = ic->conn;  in rds_ib_tasklet_fn_recv()
285 	struct rds_ib_device *rds_ibdev = ic->rds_ibdev;  in rds_ib_tasklet_fn_recv()
294 	poll_cq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);  in rds_ib_tasklet_fn_recv()
295 	ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);  in rds_ib_tasklet_fn_recv()
296 	poll_cq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);  in rds_ib_tasklet_fn_recv()
299 		rds_ib_set_ack(ic, state.ack_next, state.ack_required);  in rds_ib_tasklet_fn_recv()
300 	if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {  in rds_ib_tasklet_fn_recv()
302 		ic->i_ack_recv = state.ack_recv;  in rds_ib_tasklet_fn_recv()
306 		rds_ib_attempt_ack(ic);  in rds_ib_tasklet_fn_recv()
312 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_qp_event_handler()  local
314 	rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event,  in rds_ib_qp_event_handler()
319 		rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);  in rds_ib_qp_event_handler()
334 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cq_comp_handler_send()  local
340 	tasklet_schedule(&ic->i_send_tasklet);  in rds_ib_cq_comp_handler_send()
349 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_setup_qp()  local
350 	struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_setup_qp()
367 	if (rds_ibdev->max_wrs < ic->i_send_ring.w_nr + 1)  in rds_ib_setup_qp()
368 		rds_ib_ring_resize(&ic->i_send_ring, rds_ibdev->max_wrs - 1);  in rds_ib_setup_qp()
369 	if (rds_ibdev->max_wrs < ic->i_recv_ring.w_nr + 1)  in rds_ib_setup_qp()
370 		rds_ib_ring_resize(&ic->i_recv_ring, rds_ibdev->max_wrs - 1);  in rds_ib_setup_qp()
373 	ic->i_pd = rds_ibdev->pd;  in rds_ib_setup_qp()
375 	cq_attr.cqe = ic->i_send_ring.w_nr + 1;  in rds_ib_setup_qp()
377 	ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,  in rds_ib_setup_qp()
380 	if (IS_ERR(ic->i_send_cq)) {  in rds_ib_setup_qp()
381 		ret = PTR_ERR(ic->i_send_cq);  in rds_ib_setup_qp()
382 		ic->i_send_cq = NULL;  in rds_ib_setup_qp()
387 	cq_attr.cqe = ic->i_recv_ring.w_nr;  in rds_ib_setup_qp()
388 	ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,  in rds_ib_setup_qp()
391 	if (IS_ERR(ic->i_recv_cq)) {  in rds_ib_setup_qp()
392 		ret = PTR_ERR(ic->i_recv_cq);  in rds_ib_setup_qp()
393 		ic->i_recv_cq = NULL;  in rds_ib_setup_qp()
398 	ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);  in rds_ib_setup_qp()
404 	ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);  in rds_ib_setup_qp()
415 	attr.cap.max_send_wr = ic->i_send_ring.w_nr + 1;  in rds_ib_setup_qp()
416 	attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;  in rds_ib_setup_qp()
421 	attr.send_cq = ic->i_send_cq;  in rds_ib_setup_qp()
422 	attr.recv_cq = ic->i_recv_cq;  in rds_ib_setup_qp()
428 	ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);  in rds_ib_setup_qp()
434 	ic->i_send_hdrs = ib_dma_alloc_coherent(dev,  in rds_ib_setup_qp()
435 					   ic->i_send_ring.w_nr *  in rds_ib_setup_qp()
437 					   &ic->i_send_hdrs_dma, GFP_KERNEL);  in rds_ib_setup_qp()
438 	if (!ic->i_send_hdrs) {  in rds_ib_setup_qp()
444 	ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,  in rds_ib_setup_qp()
445 					   ic->i_recv_ring.w_nr *  in rds_ib_setup_qp()
447 					   &ic->i_recv_hdrs_dma, GFP_KERNEL);  in rds_ib_setup_qp()
448 	if (!ic->i_recv_hdrs) {  in rds_ib_setup_qp()
454 	ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),  in rds_ib_setup_qp()
455 				       &ic->i_ack_dma, GFP_KERNEL);  in rds_ib_setup_qp()
456 	if (!ic->i_ack) {  in rds_ib_setup_qp()
462 	ic->i_sends = vzalloc_node(ic->i_send_ring.w_nr * sizeof(struct rds_ib_send_work),  in rds_ib_setup_qp()
464 	if (!ic->i_sends) {  in rds_ib_setup_qp()
470 	ic->i_recvs = vzalloc_node(ic->i_recv_ring.w_nr * sizeof(struct rds_ib_recv_work),  in rds_ib_setup_qp()
472 	if (!ic->i_recvs) {  in rds_ib_setup_qp()
478 	rds_ib_recv_init_ack(ic);  in rds_ib_setup_qp()
480 	rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd,  in rds_ib_setup_qp()
481 		 ic->i_send_cq, ic->i_recv_cq);  in rds_ib_setup_qp()
537 	struct rds_ib_connection *ic = NULL;  in rds_ib_cm_handle_connect()  local
583 	ic = conn->c_transport_data;  in rds_ib_cm_handle_connect()
594 	BUG_ON(ic->i_cm_id);  in rds_ib_cm_handle_connect()
596 	ic->i_cm_id = cm_id;  in rds_ib_cm_handle_connect()
630 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_cm_initiate_connect()  local
638 	ic->i_flowctl = rds_ib_sysctl_flow_control;	/* advertise flow control */  in rds_ib_cm_initiate_connect()
657 		if (ic->i_cm_id == cm_id)  in rds_ib_cm_initiate_connect()
665 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_conn_connect()  local
671 	ic->i_cm_id = rdma_create_id(&init_net, rds_rdma_cm_event_handler, conn,  in rds_ib_conn_connect()
673 	if (IS_ERR(ic->i_cm_id)) {  in rds_ib_conn_connect()
674 		ret = PTR_ERR(ic->i_cm_id);  in rds_ib_conn_connect()
675 		ic->i_cm_id = NULL;  in rds_ib_conn_connect()
680 	rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);  in rds_ib_conn_connect()
690 	ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,  in rds_ib_conn_connect()
694 		rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,  in rds_ib_conn_connect()
696 		rdma_destroy_id(ic->i_cm_id);  in rds_ib_conn_connect()
697 		ic->i_cm_id = NULL;  in rds_ib_conn_connect()
711 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_conn_shutdown()  local
714 	rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,  in rds_ib_conn_shutdown()
715 		 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,  in rds_ib_conn_shutdown()
716 		 ic->i_cm_id ? ic->i_cm_id->qp : NULL);  in rds_ib_conn_shutdown()
718 	if (ic->i_cm_id) {  in rds_ib_conn_shutdown()
719 		struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_conn_shutdown()
721 		rdsdebug("disconnecting cm %p\n", ic->i_cm_id);  in rds_ib_conn_shutdown()
722 		err = rdma_disconnect(ic->i_cm_id);  in rds_ib_conn_shutdown()
728 				ic->i_cm_id, err);  in rds_ib_conn_shutdown()
741 			   rds_ib_ring_empty(&ic->i_recv_ring) &&  in rds_ib_conn_shutdown()
742 			   (atomic_read(&ic->i_signaled_sends) == 0));  in rds_ib_conn_shutdown()
743 		tasklet_kill(&ic->i_send_tasklet);  in rds_ib_conn_shutdown()
744 		tasklet_kill(&ic->i_recv_tasklet);  in rds_ib_conn_shutdown()
747 		if (ic->i_cm_id->qp)  in rds_ib_conn_shutdown()
748 			rdma_destroy_qp(ic->i_cm_id);  in rds_ib_conn_shutdown()
749 		if (ic->i_send_cq)  in rds_ib_conn_shutdown()
750 			ib_destroy_cq(ic->i_send_cq);  in rds_ib_conn_shutdown()
751 		if (ic->i_recv_cq)  in rds_ib_conn_shutdown()
752 			ib_destroy_cq(ic->i_recv_cq);  in rds_ib_conn_shutdown()
755 		if (ic->i_send_hdrs)  in rds_ib_conn_shutdown()
757 					   ic->i_send_ring.w_nr *  in rds_ib_conn_shutdown()
759 					   ic->i_send_hdrs,  in rds_ib_conn_shutdown()
760 					   ic->i_send_hdrs_dma);  in rds_ib_conn_shutdown()
762 		if (ic->i_recv_hdrs)  in rds_ib_conn_shutdown()
764 					   ic->i_recv_ring.w_nr *  in rds_ib_conn_shutdown()
766 					   ic->i_recv_hdrs,  in rds_ib_conn_shutdown()
767 					   ic->i_recv_hdrs_dma);  in rds_ib_conn_shutdown()
769 		if (ic->i_ack)  in rds_ib_conn_shutdown()
771 					     ic->i_ack, ic->i_ack_dma);  in rds_ib_conn_shutdown()
773 		if (ic->i_sends)  in rds_ib_conn_shutdown()
774 			rds_ib_send_clear_ring(ic);  in rds_ib_conn_shutdown()
775 		if (ic->i_recvs)  in rds_ib_conn_shutdown()
776 			rds_ib_recv_clear_ring(ic);  in rds_ib_conn_shutdown()
778 		rdma_destroy_id(ic->i_cm_id);  in rds_ib_conn_shutdown()
783 		if (ic->rds_ibdev)  in rds_ib_conn_shutdown()
784 			rds_ib_remove_conn(ic->rds_ibdev, conn);  in rds_ib_conn_shutdown()
786 		ic->i_cm_id = NULL;  in rds_ib_conn_shutdown()
787 		ic->i_pd = NULL;  in rds_ib_conn_shutdown()
788 		ic->i_send_cq = NULL;  in rds_ib_conn_shutdown()
789 		ic->i_recv_cq = NULL;  in rds_ib_conn_shutdown()
790 		ic->i_send_hdrs = NULL;  in rds_ib_conn_shutdown()
791 		ic->i_recv_hdrs = NULL;  in rds_ib_conn_shutdown()
792 		ic->i_ack = NULL;  in rds_ib_conn_shutdown()
794 	BUG_ON(ic->rds_ibdev);  in rds_ib_conn_shutdown()
797 	if (ic->i_data_op) {  in rds_ib_conn_shutdown()
800 		rm = container_of(ic->i_data_op, struct rds_message, data);  in rds_ib_conn_shutdown()
802 		ic->i_data_op = NULL;  in rds_ib_conn_shutdown()
806 	clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);  in rds_ib_conn_shutdown()
808 	atomic64_set(&ic->i_ack_next, 0);  in rds_ib_conn_shutdown()
810 	ic->i_ack_next = 0;  in rds_ib_conn_shutdown()
812 	ic->i_ack_recv = 0;  in rds_ib_conn_shutdown()
815 	ic->i_flowctl = 0;  in rds_ib_conn_shutdown()
816 	atomic_set(&ic->i_credits, 0);  in rds_ib_conn_shutdown()
818 	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);  in rds_ib_conn_shutdown()
819 	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);  in rds_ib_conn_shutdown()
821 	if (ic->i_ibinc) {  in rds_ib_conn_shutdown()
822 		rds_inc_put(&ic->i_ibinc->ii_inc);  in rds_ib_conn_shutdown()
823 		ic->i_ibinc = NULL;  in rds_ib_conn_shutdown()
826 	vfree(ic->i_sends);  in rds_ib_conn_shutdown()
827 	ic->i_sends = NULL;  in rds_ib_conn_shutdown()
828 	vfree(ic->i_recvs);  in rds_ib_conn_shutdown()
829 	ic->i_recvs = NULL;  in rds_ib_conn_shutdown()
834 	struct rds_ib_connection *ic;  in rds_ib_conn_alloc()  local
839 	ic = kzalloc(sizeof(struct rds_ib_connection), gfp);  in rds_ib_conn_alloc()
840 	if (!ic)  in rds_ib_conn_alloc()
843 	ret = rds_ib_recv_alloc_caches(ic);  in rds_ib_conn_alloc()
845 		kfree(ic);  in rds_ib_conn_alloc()
849 	INIT_LIST_HEAD(&ic->ib_node);  in rds_ib_conn_alloc()
850 	tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send,  in rds_ib_conn_alloc()
851 		     (unsigned long)ic);  in rds_ib_conn_alloc()
852 	tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv,  in rds_ib_conn_alloc()
853 		     (unsigned long)ic);  in rds_ib_conn_alloc()
854 	mutex_init(&ic->i_recv_mutex);  in rds_ib_conn_alloc()
856 	spin_lock_init(&ic->i_ack_lock);  in rds_ib_conn_alloc()
858 	atomic_set(&ic->i_signaled_sends, 0);  in rds_ib_conn_alloc()
864 	rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);  in rds_ib_conn_alloc()
865 	rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);  in rds_ib_conn_alloc()
867 	ic->conn = conn;  in rds_ib_conn_alloc()
868 	conn->c_transport_data = ic;  in rds_ib_conn_alloc()
871 	list_add_tail(&ic->ib_node, &ib_nodev_conns);  in rds_ib_conn_alloc()
884 	struct rds_ib_connection *ic = arg;  in rds_ib_conn_free()  local
887 	rdsdebug("ic %p\n", ic);  in rds_ib_conn_free()
894 	lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;  in rds_ib_conn_free()
897 	list_del(&ic->ib_node);  in rds_ib_conn_free()
900 	rds_ib_recv_free_caches(ic);  in rds_ib_conn_free()
902 	kfree(ic);  in rds_ib_conn_free()